blob: 96c4fc5da1f1a51ff364453485c57eea98d3c104 [file]
# Copyright 2026 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import collections
import concurrent.futures
import json
import os
import pathlib
import re
import statistics
import subprocess
import tempfile
import urllib.parse
from collections.abc import Collection, Set
from typing import Any, Callable, TextIO, TypeVar
_DEFAULT_STATE_FILE = (
"/google/data/ro/teams/tq-performance/sample_builds_state.json"
)
# Matches the links to each test task's CAS outputs in build step summaries,
# capturing the CAS instance and the root digest of the task's output tree.
# Step summaries also link to other CAS uploads (such as the aggregated perf
# dataset), which this deliberately does not match.
_TASK_OUTPUTS_LINK_REGEX = re.compile(
r"\[task outputs\]\(https://cas-viewer\.(?:appspot\.com|api\.luci\.app)/"
r"(projects/[^/]+/instances/[^/]+)/blobs/([0-9a-fA-F]+/[0-9]+)/tree\)"
)
_PERF_RESULTS_SUFFIX = ".fuchsiaperf.json"
# Test tasks write their results under this directory. The aggregated perf
# dataset stores each task's results without this prefix.
_TASK_RESULTS_PREFIX = "out/"
# Maximum number of concurrent `cas` invocations.
_MAX_CAS_JOBS = 8
T = TypeVar("T")
R = TypeVar("R")
def _normalize_build_id(build_id: str) -> str:
"""Normalizes a Buildbucket build ID by stripping any leading 'b' or 'B' prefix."""
return build_id.strip().removeprefix("b").removeprefix("B")
def download_metrics(args, out_fh: TextIO) -> None:
"""Downloads metrics artifacts from CAS."""
bb_tool = _find_bb_tool()
_check_bb_auth(bb_tool)
_download_metrics(bb_tool, args.build_id, args.out_dir, out_fh)
def download_baseline(args, out_fh: TextIO) -> None:
"""Finds the most recent successful sample build for a given builder and downloads its metrics."""
bb_tool = _find_bb_tool()
_check_bb_auth(bb_tool)
_download_baseline(
bb_tool,
args.builder_name,
args.count,
args.out_dir,
args.state_file,
out_fh,
)
def extract_metrics(args, out_fh: TextIO) -> None:
"""Extracts metrics from Fuchsia performance results."""
_extract_metrics(args.suite_prefix, args.data_dir, args.out_file, out_fh)
# TODO(https://github.com/python/cpython/issues/101503): Fix the visibility
# violation by changing the type specification of "subparsers" to whatever
# publicly-available type is determined for it to have.
def register_subparsers(
subparsers: argparse._SubParsersAction, out_fh: TextIO
) -> None:
"""Registers subparsers for workflow commands."""
subparser = subparsers.add_parser(
"download_metrics",
help="Downloads metrics artifacts from CAS.",
)
subparser.add_argument(
"build_id",
help="The Buildbucket build ID for which to download metrics (with or without 'b' prefix).",
)
subparser.add_argument(
"out_dir",
help="The directory to save the downloaded CAS contents into.",
)
subparser.set_defaults(func=lambda args: download_metrics(args, out_fh))
subparser = subparsers.add_parser(
"download_baseline",
help="Finds the most recent successful sample build and downloads its metrics.",
)
subparser.add_argument(
"-n",
"--count",
type=int,
default=5,
help="The number of builds to check (default: 5).",
)
subparser.add_argument(
"builder_name",
help="The name of the builder by which to filter.",
)
subparser.add_argument(
"out_dir",
help="The directory to save the downloaded CAS contents into.",
)
subparser.add_argument(
"--state_file",
default=_DEFAULT_STATE_FILE,
help="Path to the sample builds state file.",
)
subparser.set_defaults(func=lambda args: download_baseline(args, out_fh))
subparser = subparsers.add_parser(
"extract_metrics",
help="Extracts metrics from Fuchsia performance results.",
)
subparser.add_argument(
"suite_prefix",
help="The test suite prefix by which to filter metrics.",
)
subparser.add_argument(
"data_dir",
nargs="?",
default=".",
help="Directory to process. Defaults to the current directory.",
)
subparser.add_argument(
"--out_file",
help="A path to save the resulting CSV.",
)
subparser.set_defaults(func=lambda args: extract_metrics(args, out_fh))
def _download_metrics(
bb_tool: pathlib.Path,
build_id: str,
out_dir: str,
out_fh: TextIO,
) -> None:
build_id = _normalize_build_id(build_id)
os.makedirs(out_dir, exist_ok=True)
cas_tool = _find_cas_tool()
try:
build_info = _get_build_info(bb_tool, build_id, out_fh)
# Every build that ran tests has output properties, so a KeyError
# here means we cannot proceed. It is reported by the handler below.
properties = build_info["output"]["properties"]
perf_dataset = _get_perf_dataset(properties)
if perf_dataset is not None:
instance, digest = perf_dataset
_download_from_cas(cas_tool, instance, digest, out_dir, out_fh)
else:
# Builds do not publish the aggregated dataset if any test
# failed, but the results from each test task are still in CAS.
out_fh.write(
f"Build {build_id} has no aggregated perf dataset. "
"Collecting results from individual test tasks...\n"
)
failed_tests = properties.get("failed_test_names", [])
if failed_tests:
out_fh.write("Excluding results from failed tests:\n")
for test in failed_tests:
out_fh.write(f" {test}\n")
_download_task_results(
cas_tool,
build_id,
# Buildbucket omits "steps" when it is empty.
build_info.get("steps", []),
set(failed_tests),
out_dir,
out_fh,
)
except (subprocess.CalledProcessError, RuntimeError, KeyError) as e:
raise RuntimeError(f"Download failed for build {build_id}") from e
def _download_baseline(
bb_tool: str,
builder_name: str,
count: int,
out_dir: str,
state_file: str,
out_fh: TextIO,
) -> None:
if not os.path.exists(state_file):
raise FileNotFoundError(f"Error: State file not found at {state_file}")
with open(state_file, "r") as f:
state = json.load(f)
filtered_dates = [
date
for date, entry in state.items()
if isinstance(entry.get("launched"), dict)
]
sorted_dates = sorted(filtered_dates, reverse=True)
last_n_dates = sorted_dates[:count]
out_fh.write(
f"Finding most recent successful sample build for {builder_name} "
f"(checking last {count} entries)...\n"
)
for date in last_n_dates:
launched = state[date].get("launched", {})
build_id = launched.get(builder_name, {}).get("build_id", "")
if build_id:
status = _get_build_status(bb_tool, build_id)
if status == "SUCCESS":
out_fh.write(
f" {date}: {build_id} (SUCCESS) - Triggering download to {out_dir}...\n"
)
_download_metrics(bb_tool, build_id, out_dir, out_fh)
break
out_fh.write(f" {date}: {build_id} ({status}) - Skipping...\n")
else:
out_fh.write(
f"No successful sample build IDs found for {builder_name} in the last "
f"{count} entries.\n"
)
def _extract_metrics(
suite_prefix: str, data_dir: str, out_file: str, out_fh: TextIO
) -> None:
target_dir = pathlib.Path(data_dir)
perfcompare_dirs = (target_dir / "with_cl", target_dir / "without_cl")
if all(p.is_dir() for p in perfcompare_dirs):
if not out_file:
raise ValueError(
"Error: --out_file is required when data_dir contains results from multiple builds."
)
with open(perfcompare_dirs[0] / out_file, "w") as f:
_process_files(suite_prefix, perfcompare_dirs[0], f, out_fh)
with open(perfcompare_dirs[1] / out_file, "w") as f:
_process_files(suite_prefix, perfcompare_dirs[1], f, out_fh)
elif out_file:
with open(out_file, "w") as f:
_process_files(suite_prefix, target_dir, f, out_fh)
else:
_process_files(suite_prefix, target_dir, out_fh, out_fh)
def _process_files(
suite_prefix: str,
search_dir: pathlib.Path,
out_file: TextIO,
out_fh: TextIO,
) -> None:
"""Finds and processes all *.fuchsiaperf.json files in a single pass."""
fuchsia_perf_files = sorted(search_dir.rglob("*.fuchsiaperf.json"))
aggregated_results = collections.defaultdict(
lambda: collections.defaultdict(list)
)
for file_path in fuchsia_perf_files:
try:
with open(file_path) as f:
data = json.load(f)
for record in data:
suite = record.get("test_suite")
label = record.get("label")
if suite and suite.startswith(suite_prefix):
# We just take the mean if there are multiple values.
# This is lossy, but it's the most sensical thing we can
# do when aggregating multiple runs for output in CSV.
value = statistics.mean(record["values"])
aggregated_results[suite][label].append(value)
except json.JSONDecodeError as e:
raise RuntimeError(
f"Malformed JSON in file {file_path}: {e}"
) from e
for suite, metrics in sorted(aggregated_results.items()):
for metric, values in sorted(metrics.items()):
values_str = ",".join(map(str, values))
out_file.write(f"{suite},{metric},{values_str}\n")
def _get_build_info(
bb_tool: pathlib.Path, build_id: str, out_fh: TextIO
) -> dict[str, Any]:
"""Uses the `bb` tool to get a build's output properties and steps."""
cmd = [
bb_tool,
"get",
"-fields",
"output.properties,steps",
"-json",
build_id,
]
out_fh.write(f"Fetching build info for {build_id} using bb...\n")
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
try:
return json.loads(result.stdout)
except json.JSONDecodeError as e:
raise RuntimeError(f"Failed to parse bb output as JSON: {e}") from e
def _get_perf_dataset(properties: dict[str, Any]) -> tuple[str, str] | None:
"""Returns the (CAS instance, digest) of a build's aggregated perf dataset.
Returns None if the build did not publish an aggregated dataset.
"""
digest = properties.get("perf_dataset_digest") or properties.get(
"perfcompare_dataset_digest"
)
if not digest:
return None
return properties["cas_instance"], digest
def _download_from_cas(
cas_tool: pathlib.Path,
instance: str,
digest: str,
out_dir: str,
out_fh: TextIO,
) -> None:
"""Uses the hermetic `cas` binary to download the specified digest."""
cmd = [
cas_tool,
"download",
"-digest",
digest,
"-cas-instance",
instance,
"-dir",
out_dir,
]
out_fh.write(f"Downloading from cas to {out_dir}...\n")
subprocess.run(cmd, check=True)
def _extract_task_outputs_from_steps(
steps: Collection[dict[str, Any]],
) -> list[tuple[str, str]]:
"""Extracts CAS (instance, digest) pairs of test task outputs.
Returns them sorted so that boot directories are numbered
deterministically. A task linked from more than one step is returned
once, so that its results are not counted twice.
"""
return sorted(
{
(match.group(1), match.group(2))
for step in steps
for match in _TASK_OUTPUTS_LINK_REGEX.finditer(
step.get("summaryMarkdown", "")
)
}
)
def _download_task_results(
cas_tool: pathlib.Path,
build_id: str,
steps: Collection[dict[str, Any]],
failed_tests: Set[str],
out_dir: str,
out_fh: TextIO,
) -> None:
"""Downloads perf results from a build's test tasks into out_dir/by_boot.
This reproduces the layout of the aggregated perf dataset: each task that
produced perf results gets its own boot directory, holding only its
*.fuchsiaperf.json files from passing tests. Other task outputs (traces,
logs, snapshots) can be gigabytes per task and are not downloaded.
"""
task_outputs = _extract_task_outputs_from_steps(steps)
if not task_outputs:
raise RuntimeError(
f"No task outputs found in build steps for {build_id}"
)
by_boot_dir = os.path.join(out_dir, "by_boot")
# Check this before downloading anything, rather than letting the
# os.rename() below fail after all the downloads.
if os.path.lexists(by_boot_dir):
raise RuntimeError(
f"{by_boot_dir} already exists. "
"Use an empty output directory to avoid mixing results."
)
out_fh.write(
f"Listing perf results in {len(task_outputs)} task outputs...\n"
)
results_per_task = _parallel_map(
lambda task: _list_perf_results(
cas_tool, task[0], task[1], failed_tests
),
task_outputs,
)
boots = [
(instance, results)
for (instance, _), results in zip(task_outputs, results_per_task)
if results
]
if not boots:
raise RuntimeError(
f"No {_PERF_RESULTS_SUFFIX} files found in task outputs for "
f"build {build_id}"
)
# Download into a staging directory so that a failed download does not
# leave an incomplete by_boot directory that looks like a valid dataset.
with tempfile.TemporaryDirectory(
prefix=".by_boot.", dir=out_dir
) as staging_dir:
staged_by_boot_dir = os.path.join(staging_dir, "by_boot")
os.mkdir(staged_by_boot_dir)
downloads = []
for boot_idx, (instance, results) in enumerate(boots):
boot_dir = os.path.join(staged_by_boot_dir, f"boot{boot_idx:06d}")
for rel_path, digest in results:
downloads.append(
(instance, digest, os.path.join(boot_dir, rel_path))
)
out_fh.write(
f"Downloading {len(downloads)} perf results files from "
f"{len(boots)} tasks to {by_boot_dir}...\n"
)
_parallel_map(
lambda download: _download_file_from_cas(
cas_tool, download[0], download[1], download[2]
),
downloads,
)
os.rename(staged_by_boot_dir, by_boot_dir)
def _list_perf_results(
cas_tool: pathlib.Path,
instance: str,
digest: str,
failed_tests: Set[str],
) -> list[tuple[str, str]]:
"""Lists the perf results files in a task's CAS output tree.
Returns sorted (path, file digest) pairs, with paths relative to the
task's results directory, excluding results from failed_tests.
"""
cmd = [cas_tool, "tree", "-cas-instance", instance, "-digest", digest]
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
tree = json.loads(result.stdout)
# testrunner names each test's output directory after the test with
# colons removed and then URL-escaped (see Record() in
# //tools/testing/testrunner/outputs.go). Unescaping cannot restore the
# colons, so remove them from the failed test names before comparing.
failed_tests_munged = {name.replace(":", "") for name in failed_tests}
results = []
for path, entry in tree["files"].items():
if not path.endswith(_PERF_RESULTS_SUFFIX):
continue
rel_path = path.removeprefix(_TASK_RESULTS_PREFIX)
test_name = urllib.parse.unquote(rel_path.split("/", 1)[0])
if test_name not in failed_tests_munged:
results.append((rel_path, entry["digest"]))
return sorted(results)
def _download_file_from_cas(
cas_tool: pathlib.Path, instance: str, digest: str, dest: str
) -> None:
"""Uses the hermetic `cas` binary to download a single file."""
os.makedirs(os.path.dirname(dest), exist_ok=True)
cmd = [
cas_tool,
"download",
"-cas-instance",
instance,
"-digest",
digest,
"-file",
dest,
]
subprocess.run(cmd, check=True)
def _parallel_map(fn: Callable[[T], R], items: Collection[T]) -> list[R]:
"""Returns [fn(item) for item in items], running calls concurrently.
If any call raises, calls that have not started are cancelled, and the
exception is re-raised once the running calls finish.
"""
with concurrent.futures.ThreadPoolExecutor(_MAX_CAS_JOBS) as executor:
futures = [executor.submit(fn, item) for item in items]
try:
for future in concurrent.futures.as_completed(futures):
future.result()
return [future.result() for future in futures]
finally:
for future in futures:
future.cancel()
def _get_build_status(bb_tool: pathlib.Path, build_id: str) -> str:
"""Uses the `bb` tool to get the status of a build."""
build_id = _normalize_build_id(build_id)
cmd = [bb_tool, "get", "-json", build_id]
try:
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
build_info = json.loads(result.stdout)
return build_info.get("status", "UNKNOWN")
except (subprocess.CalledProcessError, json.JSONDecodeError):
return "ERROR"
def _check_bb_auth(bb_tool: pathlib.Path) -> None:
"""Checks if the user is authenticated with buildbucket."""
result = subprocess.run(
[bb_tool, "auth-info"], capture_output=True, text=True
)
if result.returncode != 0:
raise RuntimeError(
'Error: "bb auth-info" returned an error, indicating that you are '
"not authenticated with Buildbucket.\n"
f"Please run: {bb_tool} auth-login"
)
def _find_bb_tool() -> pathlib.Path:
"""Finds the path to the hermetic `bb` binary in the Fuchsia checkout."""
fuchsia_dir = _ensure_fuchsia_dir()
bb_path = fuchsia_dir / "prebuilt" / "tools" / "buildbucket" / "bb"
if not bb_path.exists():
raise RuntimeError(
f"Could not find `bb` tool at expected location: {bb_path}"
)
return bb_path
def _find_cas_tool() -> pathlib.Path:
"""Finds the path to the hermetic `cas` binary in the Fuchsia checkout."""
fuchsia_dir = _ensure_fuchsia_dir()
cas_path = fuchsia_dir / "prebuilt" / "tools" / "cas" / "cas"
if not cas_path.exists():
raise RuntimeError(
f"Could not find `cas` tool at expected location: {cas_path}"
)
return cas_path
def _ensure_fuchsia_dir() -> pathlib.Path:
fuchsia_dir = os.environ.get("FUCHSIA_DIR")
if not fuchsia_dir:
raise RuntimeError(
"FUCHSIA_DIR environment variable is not set. Are you running via `fx`?"
)
return pathlib.Path(fuchsia_dir)