blob: 38497e1ad5b67f9efc7563b51622567793b4b4d5 [file]
# Copyright 2026 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Fuchsia Python coverage runtime collector.
Activated when the FUCHSIA_PYTHON_COVERAGE_DIR environment variable is set to a
directory path. Collects line coverage using coverage.py and writes a compact
JSON file per process on exit for merging by covargs.
"""
import atexit
import json
import os
import sys
import uuid
import zipfile
from collections.abc import Mapping
_cov = None
_srcmap: dict[str, str] = {}
_dumped = False
_pyz_archive: str | None = None
def _find_pyz_archive(path: str) -> tuple[str, str] | None:
low = path.lower()
idx = low.find(".pyz" + os.sep)
if idx == -1 and os.altsep:
idx = low.find(".pyz" + os.altsep)
if idx >= 0:
return path[: idx + 4], path[idx + 5 :]
return None
def _patch_coverage_pyz(
coverage_files: object, coverage_python: object
) -> None:
orig_zip_location = getattr(coverage_files, "zip_location", None)
if orig_zip_location is None or getattr(
orig_zip_location, "_fuchsia_patched", False
):
return
def _zip_location_with_pyz(filename: str) -> tuple[str, str] | None:
res = orig_zip_location(filename)
if res is not None:
return res
pyz_parts = _find_pyz_archive(filename)
if pyz_parts is not None:
zippath, inner = pyz_parts
if os.path.isfile(zippath):
return (zippath, inner)
return None
setattr(_zip_location_with_pyz, "_fuchsia_patched", True)
setattr(coverage_files, "zip_location", _zip_location_with_pyz)
setattr(coverage_python, "zip_location", _zip_location_with_pyz)
def _init_pyz_state() -> None:
"""Discovers the running .pyz archive and loads its bundled source map."""
global _pyz_archive, _srcmap
if _pyz_archive is not None and _srcmap:
return
pyz_parts = _find_pyz_archive(os.path.abspath(__file__))
candidates = (
*((pyz_parts[0],) if pyz_parts is not None else ()),
*(
os.path.abspath(p)
for p in (*sys.path, *sys.argv[:1])
if p.lower().endswith(".pyz")
),
)
fallback_pyz = _pyz_archive
for candidate in candidates:
if not os.path.isfile(candidate):
continue
if fallback_pyz is None:
fallback_pyz = candidate
try:
with zipfile.ZipFile(candidate, "r") as zf:
_srcmap = json.loads(
zf.read("_fuchsia_pycov_srcmap.json").decode("utf-8")
)
_pyz_archive = candidate
return
except KeyError:
pass
except (OSError, zipfile.BadZipFile, json.JSONDecodeError, ValueError):
pass
if _pyz_archive is None:
_pyz_archive = fallback_pyz
def _child_site_entries(cov_dir: str) -> tuple[str, ...]:
fuchsia_dir = os.environ.get("FUCHSIA_DIR")
if fuchsia_dir:
site_dir = os.path.join(fuchsia_dir, "build", "python", "coverage_site")
cov_src_dir = os.path.join(
fuchsia_dir, "third_party", "pylibs", "coverage", "src"
)
if os.path.isdir(site_dir) and os.path.isdir(cov_src_dir):
return (site_dir, cov_src_dir)
if not _pyz_archive or not os.path.isfile(_pyz_archive):
return ()
helper_zip = os.path.join(cov_dir, "_pycov_site.zip")
if not os.path.exists(helper_zip):
tmp_zip = os.path.join(cov_dir, f".tmp_pycov_site_{os.getpid()}.zip")
try:
os.makedirs(cov_dir, exist_ok=True)
wrote_sitecustomize = False
wrote_pycov = False
wrote_coverage_init = False
with (
zipfile.ZipFile(_pyz_archive, "r") as src_zf,
zipfile.ZipFile(
tmp_zip, "w", compression=zipfile.ZIP_DEFLATED
) as dst_zf,
):
for name in src_zf.namelist():
base = os.path.basename(name)
if name in (
"sitecustomize.py",
"_fuchsia_pycov.py",
"coverage_site/sitecustomize.py",
"coverage_site/_fuchsia_pycov.py",
):
dst_zf.writestr(base, src_zf.read(name))
if base == "sitecustomize.py":
wrote_sitecustomize = True
elif base == "_fuchsia_pycov.py":
wrote_pycov = True
elif name.startswith("coverage/"):
dst_zf.writestr(name, src_zf.read(name))
if name == "coverage/__init__.py":
wrote_coverage_init = True
if wrote_sitecustomize and wrote_pycov and wrote_coverage_init:
os.replace(tmp_zip, helper_zip)
except (OSError, zipfile.BadZipFile) as err:
print(
f"WARNING: _fuchsia_pycov failed to write {helper_zip}: {err}",
file=sys.stderr,
)
finally:
if os.path.exists(tmp_zip):
try:
os.remove(tmp_zip)
except OSError:
pass
return (helper_zip,) if os.path.exists(helper_zip) else ()
def _ensure_child_site_env(cov_dir: str) -> None:
"""Ensures child sys.executable subprocesses inherit coverage hooks."""
entries = _child_site_entries(cov_dir)
if not entries:
return
cur_parts = tuple(
p for p in os.environ.get("PYTHONPATH", "").split(os.pathsep) if p
)
missing = tuple(p for p in entries if p not in cur_parts)
if missing:
os.environ["PYTHONPATH"] = os.pathsep.join((*missing, *cur_parts))
def start() -> None:
global _cov
cov_dir = os.environ.get("FUCHSIA_PYTHON_COVERAGE_DIR")
if not cov_dir:
return
cov_dir = os.path.abspath(cov_dir)
os.environ["FUCHSIA_PYTHON_COVERAGE_DIR"] = cov_dir
# Skip test listing invocations that inspect test cases without running them.
if any(
arg
in (
"list_host_python_unittests",
"--list_host_python_unittests",
"--list_tests",
"list_mobly_tests",
)
for arg in sys.argv
):
return
# Normalize .pyz entries on sys.path (and sys.argv[0] before CPython adds
# it to sys.path[0]) as well as environment paths so resolution remains
# valid even if the test calls os.chdir().
if (
sys.argv
and sys.argv[0].lower().endswith(".pyz")
and not os.path.isabs(sys.argv[0])
):
sys.argv[0] = os.path.abspath(sys.argv[0])
for i, p in enumerate(sys.path):
if (
p.lower().endswith(".pyz") or _find_pyz_archive(p) is not None
) and not os.path.isabs(p):
sys.path[i] = os.path.abspath(p)
for env_key in ("FUCHSIA_DIR", "RUNFILES_DIR", "RUNFILES_MANIFEST_FILE"):
env_val = os.environ.get(env_key)
if env_val and not os.path.isabs(env_val):
os.environ[env_key] = os.path.abspath(env_val)
_init_pyz_state()
_ensure_child_site_env(cov_dir)
if _cov is not None:
return
try:
import coverage
import coverage.exceptions
import coverage.files
import coverage.python
_patch_coverage_pyz(coverage.files, coverage.python)
omit_patterns = (
"*lib_python*.zip/*",
"*_pycov_site.zip/*",
"*/third_party/*",
"*/prebuilt/*",
"*_stage2_bootstrap.py",
"*_bazel_site_init.py",
"*_fuchsia_pycov.py",
"*/sitecustomize.py",
)
cov = coverage.Coverage(
data_file=None,
auto_data=False,
config_file=False,
branch=False,
omit=list(omit_patterns),
)
cov.start()
_cov = cov
atexit.register(stop_and_dump)
except Exception as err:
_cov = None
print(
f"WARNING: _fuchsia_pycov failed to initialize coverage: {err}",
file=sys.stderr,
)
def _is_excluded_repo_path(repo_rel: str) -> bool:
if repo_rel.startswith(
(
"out/",
"third_party/",
"prebuilt/",
"bazel-out/",
"external/",
".jiri/",
".git/",
)
):
return True
if "/third_party/" in repo_rel or "/prebuilt/" in repo_rel:
return True
return os.path.basename(repo_rel) in (
"_stage2_bootstrap.py",
"_bazel_site_init.py",
"_fuchsia_pycov.py",
"sitecustomize.py",
)
def _collect_runfiles_map() -> tuple[dict[str, str], dict[str, str]]:
"""Returns (path_to_repo_rel, repo_rel_to_target_path) for Bazel .runfiles/_main."""
path_to_rel = {}
rel_to_target = {}
runfiles_dirs = set()
env_rf = os.environ.get("RUNFILES_DIR")
if env_rf:
runfiles_dirs.add(env_rf)
env_manifest = os.environ.get("RUNFILES_MANIFEST_FILE", "")
if env_manifest.endswith(".runfiles/MANIFEST"):
runfiles_dirs.add(env_manifest[: -len("/MANIFEST")])
for rf_dir in runfiles_dirs:
rf_main = os.path.join(rf_dir, "_main")
if not os.path.isdir(rf_main):
continue
for root, dirs, files in os.walk(rf_main):
dirs[:] = [
d
for d in dirs
if d
not in (
"third_party",
"prebuilt",
"out",
"external",
".git",
"__pycache__",
)
and not d.endswith(".venv")
]
for fname in files:
if not fname.endswith(".py"):
continue
full_path = os.path.join(root, fname)
repo_rel = os.path.relpath(full_path, rf_main).replace(
"\\", "/"
)
if _is_excluded_repo_path(repo_rel):
continue
path_to_rel[os.path.abspath(full_path)] = repo_rel
real_p = os.path.realpath(full_path)
path_to_rel[real_p] = repo_rel
rel_to_target[repo_rel] = real_p
return path_to_rel, rel_to_target
def _resolve_source_path(
measured_path: str, runfiles_path_map: Mapping[str, str]
) -> str | None:
# 1. GN .pyz zipimport paths: /abs/path/to/foo.pyz/module/file.py
pyz_parts = _find_pyz_archive(measured_path)
if pyz_parts is not None:
rel_in_pyz = pyz_parts[1].replace("\\", "/")
repo_rel = _srcmap.get(rel_in_pyz)
if repo_rel and not _is_excluded_repo_path(repo_rel):
return repo_rel
return None
# 2. Bazel runfiles paths (including symlink-resolved paths)
for candidate in (
os.path.abspath(measured_path),
os.path.realpath(measured_path),
):
if candidate in runfiles_path_map:
return runfiles_path_map[candidate]
norm_path = os.path.abspath(measured_path).replace("\\", "/")
marker = ".runfiles/_main/"
marker_idx = norm_path.find(marker)
if marker_idx >= 0:
repo_rel = norm_path[marker_idx + len(marker) :]
if not _is_excluded_repo_path(repo_rel):
return repo_rel
return None
# 3. Loose files under FUCHSIA_DIR
fuchsia_dir = os.environ.get("FUCHSIA_DIR")
if fuchsia_dir:
for base_dir, candidate in (
(os.path.abspath(fuchsia_dir), os.path.abspath(measured_path)),
(os.path.realpath(fuchsia_dir), os.path.realpath(measured_path)),
):
prefix = base_dir.replace("\\", "/").rstrip("/") + "/"
norm_cand = candidate.replace("\\", "/")
if norm_cand.startswith(prefix):
repo_rel = norm_cand[len(prefix) :]
if not _is_excluded_repo_path(repo_rel):
return repo_rel
return None
return None
def stop_and_dump() -> None:
global _dumped
if _dumped or _cov is None:
return
_dumped = True
cov_dir = os.environ.get("FUCHSIA_PYTHON_COVERAGE_DIR")
if not cov_dir:
return
import coverage
try:
_cov.stop()
measured = _cov.get_data().measured_files()
except (coverage.exceptions.CoverageException, OSError) as err:
print(
f"WARNING: _fuchsia_pycov failed to collect coverage data: {err}",
file=sys.stderr,
)
return
if not measured:
return
runfiles_path_map, runfiles_rel_to_target = _collect_runfiles_map()
files_to_analyze = {}
seen_repo_rels = set()
for f in measured:
repo_rel = _resolve_source_path(f, runfiles_path_map)
if repo_rel:
files_to_analyze[f] = repo_rel
seen_repo_rels.add(repo_rel)
if not files_to_analyze:
return
# Include unimported first-party modules bundled into this .pyz so they
# appear with 0% coverage.
if _srcmap and _pyz_archive:
for rel_in_pyz, repo_rel in _srcmap.items():
if repo_rel not in seen_repo_rels and not _is_excluded_repo_path(
repo_rel
):
files_to_analyze[
os.path.join(_pyz_archive, rel_in_pyz)
] = repo_rel
seen_repo_rels.add(repo_rel)
# Include unimported first-party modules in Bazel .runfiles/_main.
for repo_rel, target_path in runfiles_rel_to_target.items():
if repo_rel not in seen_repo_rels:
files_to_analyze[target_path] = repo_rel
seen_repo_rels.add(repo_rel)
merged_by_repo: dict[str, tuple[set[int], set[int]]] = {}
for target_path, repo_rel in files_to_analyze.items():
try:
_, statements, _, missing, _ = _cov.analysis2(target_path)
except (coverage.exceptions.CoverageException, OSError):
continue
if not statements:
continue
stmt_set = set(statements)
exec_set = stmt_set - set(missing)
prev_exec, prev_stmts = merged_by_repo.get(repo_rel, (set(), set()))
merged_by_repo[repo_rel] = (prev_exec | exec_set, prev_stmts | stmt_set)
out_files = {
repo_rel: {
"executed_lines": sorted(exec_lines),
"missing_lines": sorted(stmt_lines - exec_lines),
}
for repo_rel, (exec_lines, stmt_lines) in merged_by_repo.items()
}
if not out_files:
return
tag = f"{os.getpid()}_{uuid.uuid4().hex[:8]}"
tmp_path = os.path.join(cov_dir, f".tmp_pycov_{tag}.json")
out_path = os.path.join(cov_dir, f"pycov_{tag}.json")
try:
os.makedirs(cov_dir, exist_ok=True)
with open(tmp_path, "w", encoding="utf-8") as fp:
json.dump({"files": out_files}, fp)
os.replace(tmp_path, out_path)
except (OSError, TypeError, ValueError) as err:
print(
f"WARNING: _fuchsia_pycov failed to write {out_path}: {err}",
file=sys.stderr,
)
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass