|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# |
| 3 | +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | +# See https://llvm.org/LICENSE.txt for license information. |
| 5 | +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | +# |
| 7 | +# ==------------------------------------------------------------------------==# |
| 8 | + |
| 9 | +import argparse |
| 10 | +import os |
| 11 | +import random |
| 12 | +import shutil |
| 13 | +import subprocess |
| 14 | +import tempfile |
| 15 | + |
| 16 | +# The purpose of this script is to measure the performance effect |
| 17 | +# of a PFP change in a statistically sound way, automating all the |
| 18 | +# tedious parts of doing so. It copies the test case (a Fleetbench binary) |
| 19 | +# into /tmp as well as running the test binaries from /tmp to reduce the influence on the test |
| 20 | +# machine's storage medium on the results. It accounts for measurement |
| 21 | +# bias caused by binary layout (using the --randomize-section-padding |
| 22 | +# flag to link the test binaries) and by environment variable size. |
| 23 | +# Runs of the base and test case are interleaved to account for environmental factors which may influence |
| 24 | +# the result due to the passage of time. The results |
| 25 | +# are collected into a results-base.csv and results-test.csv file in the output directory and may |
| 26 | +# be analyzed by the user with a tool such as ministat. |
| 27 | +# |
| 28 | +# Requirements: Linux host, /tmp is tmpfs. |
| 29 | +# |
| 30 | +# Example invocation for comparing the performance of the current commit |
| 31 | +# against the previous commit which is treated as the baseline, without |
| 32 | +# linking debug info: |
| 33 | +# |
| 34 | +# llvm/utils/pfp-bench \ |
| 35 | +# --base-commit HEAD^ \ |
| 36 | +# --test-commit HEAD \ |
| 37 | +# --project-path /path/to/fleetbench \ |
| 38 | +# --num-iterations 512 \ |
| 39 | +# --num-binary-variants 16 \ |
| 40 | +# --output-dir outdir |
| 41 | +# |
| 42 | +# Then this bash command will compare the real time of BM_PROTO_Arena in the |
| 43 | +# base and test cases. |
| 44 | +# |
| 45 | +# ministat -A \ |
| 46 | +# <(grep BM_PROTO_Arena results-base.csv | cut -d, -f3) \ |
| 47 | +# <(grep BM_PROTO_Arena results-test.csv | cut -d, -f3) |
| 48 | + |
| 49 | +# We don't want to copy stat() information when we copy the reproducer |
| 50 | +# to the temporary directory. Files in the Bazel store are read-only so this will |
| 51 | +# cause trouble when the linker writes the output file and when we want to clean |
| 52 | +# up the temporary directory. Python doesn't provide a way to disable copying |
| 53 | +# stat() information in shutil.copytree so we just monkeypatch shutil.copystat |
| 54 | +# to do nothing. |
| 55 | +shutil.copystat = lambda *args, **kwargs: 0 |
| 56 | + |
| 57 | +parser = argparse.ArgumentParser(prog="benchmark_change.py") |
| 58 | +parser.add_argument("--base-commit", required=True) |
| 59 | +parser.add_argument("--test-commit", required=True) |
| 60 | +parser.add_argument("--project-path", required=True) |
| 61 | +parser.add_argument("--num-iterations", type=int, required=True) |
| 62 | +parser.add_argument("--num-binary-variants", type=int, required=True) |
| 63 | +parser.add_argument("--output-dir", required=True) |
| 64 | +args = parser.parse_args() |
| 65 | + |
| 66 | +test_dir = tempfile.mkdtemp() |
| 67 | +print(f"Using {test_dir} as temporary directory") |
| 68 | + |
| 69 | +os.makedirs(args.output_dir) |
| 70 | +print(f"Using {args.output_dir} as output directory") |
| 71 | + |
| 72 | +pfp_bazel = f"{os.path.dirname(__file__)}/pfp-bazel" |
| 73 | +bazel_flags = ["-c", "opt", "--copt=-gmlt", "--verbose_failures"] |
| 74 | + |
| 75 | +def extract_link_command(target): |
| 76 | + link_command = None |
| 77 | + for line in subprocess.Popen( |
| 78 | + f"{pfp_bazel} aquery {' '.join(bazel_flags)} {target} | commandify", stdout=subprocess.PIPE, shell=True, cwd=args.project_path |
| 79 | + ).stdout.readlines(): |
| 80 | + commands = line.decode("utf-8").split("&&") |
| 81 | + for command in commands: |
| 82 | + if " -o " in command and "-fuse-ld=" in command: |
| 83 | + link_command = command.strip() |
| 84 | + return link_command |
| 85 | + |
| 86 | + |
| 87 | +def generate_binary_variants(case_name): |
| 88 | + target = "//fleetbench" |
| 89 | + subprocess.run([pfp_bazel, "clean"], cwd=args.project_path) |
| 90 | + subprocess.run([pfp_bazel, "build"] + bazel_flags + [target], cwd=args.project_path) |
| 91 | + link_command = extract_link_command(target) |
| 92 | + |
| 93 | + for i in range(0, args.num_binary_variants): |
| 94 | + print(f"Generating binary variant {i} for {case_name} case") |
| 95 | + command = f"{link_command} -o {test_dir}/fleetbench-{case_name}{i} -Wl,--randomize-section-padding={i}" |
| 96 | + subprocess.run(command, check=True, shell=True, cwd=args.project_path) |
| 97 | + command2 = f"ln -sf fleetbench.runfiles {test_dir}/fleetbench-{case_name}{i}.runfiles" |
| 98 | + subprocess.run(command2, check=True, shell=True) |
| 99 | + |
| 100 | + |
| 101 | +# Make sure that there are no local changes. |
| 102 | +subprocess.run(["git", "diff", "--exit-code", "HEAD"], check=True) |
| 103 | + |
| 104 | +# Resolve the base and test commit, since if they are relative to HEAD we will |
| 105 | +# check out the wrong commit below. |
| 106 | +resolved_base_commit = subprocess.check_output( |
| 107 | + ["git", "rev-parse", args.base_commit] |
| 108 | +).strip() |
| 109 | +resolved_test_commit = subprocess.check_output( |
| 110 | + ["git", "rev-parse", args.test_commit] |
| 111 | +).strip() |
| 112 | + |
| 113 | + |
| 114 | +subprocess.run(["git", "checkout", resolved_base_commit], check=True) |
| 115 | +generate_binary_variants("base") |
| 116 | + |
| 117 | +subprocess.run(["git", "checkout", resolved_test_commit], check=True) |
| 118 | +generate_binary_variants("test") |
| 119 | + |
| 120 | +shutil.copytree(f"{args.project_path}/bazel-bin/fleetbench/fleetbench.runfiles", f"{test_dir}/fleetbench.runfiles") |
| 121 | + |
| 122 | +def benchmark_command(case_name, binary_variant): |
| 123 | + return [f"{test_dir}/fleetbench-{case_name}{binary_variant}", "--benchmark_format=csv"] |
| 124 | + |
| 125 | + |
| 126 | +results_base_csv = open(f"{args.output_dir}/results-base.csv", "w") |
| 127 | +results_test_csv = open(f"{args.output_dir}/results-test.csv", "w") |
| 128 | + |
| 129 | +env = os.environ.copy() |
| 130 | +rng = random.Random(0) |
| 131 | + |
| 132 | +for i in range(0, args.num_iterations): |
| 133 | + for v in range(0, args.num_binary_variants): |
| 134 | + env['PAD'] = 'a' * rng.getrandbits(12) |
| 135 | + subprocess.run(benchmark_command("base", v), stdout=results_base_csv, env=env) |
| 136 | + env['PAD'] = 'a' * rng.getrandbits(12) |
| 137 | + subprocess.run(benchmark_command("test", v), stdout=results_test_csv, env=env) |
| 138 | + |
| 139 | +shutil.rmtree(test_dir) |
0 commit comments