| """Compare performance of mypyc-compiled mypy between one or more commits/branches. |
| |
| Simple usage: |
| |
| python misc/perf_compare.py my-branch master ... |
| |
| What this does: |
| |
| * Create a temp clone of the mypy repo for each target commit to measure |
| * Checkout a target commit in each of the clones |
| * Compile mypyc in each of the clones *in parallel* |
| * Create another temp clone of the mypy repo as the code to check |
| * Self check with each of the compiled mypys N times |
| * Report the average runtimes and relative performance |
| * Remove the temp clones |
| """ |
| |
| from __future__ import annotations |
| |
| import argparse |
| import glob |
| import os |
| import random |
| import shutil |
| import statistics |
| import subprocess |
| import sys |
| import threading |
| import time |
| |
| |
| def heading(s: str) -> None: |
| print() |
| print(f"=== {s} ===") |
| print() |
| |
| |
| def build_mypy(target_dir: str) -> None: |
| env = os.environ.copy() |
| env["CC"] = "clang" |
| env["MYPYC_OPT_LEVEL"] = "2" |
| cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"] |
| subprocess.run(cmd, env=env, check=True, cwd=target_dir) |
| |
| |
| def clone(target_dir: str, commit: str | None) -> None: |
| heading(f"Cloning mypy to {target_dir}") |
| repo_dir = os.getcwd() |
| if os.path.isdir(target_dir): |
| print(f"{target_dir} exists: deleting") |
| shutil.rmtree(target_dir) |
| subprocess.run(["git", "clone", repo_dir, target_dir], check=True) |
| if commit: |
| subprocess.run(["git", "checkout", commit], check=True, cwd=target_dir) |
| |
| |
| def run_benchmark(compiled_dir: str, check_dir: str) -> float: |
| cache_dir = os.path.join(compiled_dir, ".mypy_cache") |
| if os.path.isdir(cache_dir): |
| shutil.rmtree(cache_dir) |
| env = os.environ.copy() |
| env["PYTHONPATH"] = os.path.abspath(compiled_dir) |
| abschk = os.path.abspath(check_dir) |
| cmd = [ |
| sys.executable, |
| "-m", |
| "mypy", |
| "--config-file", |
| os.path.join(abschk, "mypy_self_check.ini"), |
| ] |
| cmd += glob.glob(os.path.join(abschk, "mypy/*.py")) |
| cmd += glob.glob(os.path.join(abschk, "mypy/*/*.py")) |
| t0 = time.time() |
| # Ignore errors, since some commits being measured may generate additional errors. |
| subprocess.run(cmd, cwd=compiled_dir, env=env) |
| return time.time() - t0 |
| |
| |
| def main() -> None: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("commit", nargs="+") |
| args = parser.parse_args() |
| commits = args.commit |
| num_runs = 16 |
| |
| if not (os.path.isdir(".git") and os.path.isdir("mypyc")): |
| sys.exit("error: Run this the mypy repo root") |
| |
| build_threads = [] |
| target_dirs = [] |
| for i, commit in enumerate(commits): |
| target_dir = f"mypy.{i}.tmpdir" |
| target_dirs.append(target_dir) |
| clone(target_dir, commit) |
| t = threading.Thread(target=lambda: build_mypy(target_dir)) |
| t.start() |
| build_threads.append(t) |
| |
| self_check_dir = "mypy.self.tmpdir" |
| clone(self_check_dir, commits[0]) |
| |
| heading("Compiling mypy") |
| print("(This will take a while...)") |
| |
| for t in build_threads: |
| t.join() |
| |
| print(f"Finished compiling mypy ({len(commits)} builds)") |
| |
| heading("Performing measurements") |
| |
| results: dict[str, list[float]] = {} |
| for n in range(num_runs): |
| if n == 0: |
| print("Warmup...") |
| else: |
| print(f"Run {n}/{num_runs - 1}...") |
| items = list(enumerate(commits)) |
| random.shuffle(items) |
| for i, commit in items: |
| tt = run_benchmark(target_dirs[i], self_check_dir) |
| # Don't record the first warm-up run |
| if n > 0: |
| print(f"{commit}: t={tt:.3f}s") |
| results.setdefault(commit, []).append(tt) |
| |
| print() |
| heading("Results") |
| first = -1.0 |
| for commit in commits: |
| tt = statistics.mean(results[commit]) |
| if first < 0: |
| delta = "0.0%" |
| first = tt |
| else: |
| d = (tt / first) - 1 |
| delta = f"{d:+.1%}" |
| print(f"{commit:<25} {tt:.3f}s ({delta})") |
| |
| shutil.rmtree(self_check_dir) |
| for target_dir in target_dirs: |
| shutil.rmtree(target_dir) |
| |
| |
| if __name__ == "__main__": |
| main() |