| #!/usr/bin/env python3 | |
| # Copyright 2026 The Fuchsia Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| "Implement the running of a Bazel command from Ninja and copying the outputs back into the ninja outdir." | |
| import contextlib | |
| import dataclasses | |
| import json | |
| import os | |
| import shutil | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import typing as T | |
| from pathlib import Path | |
| _SCRIPT_DIR = os.path.dirname(__file__) | |
| # LINT.IfChange(imports) | |
| from build.api.debug_symbols import ( | |
| DebugSymbolEntryType, | |
| DebugSymbolExporter, | |
| DebugSymbolsManifestParser, | |
| ) | |
| from build.bazel.scripts import ( | |
| bazel_action_utils, | |
| bazel_label_mapper, | |
| bazel_rust_analyzer_utils, | |
| bazel_source_path_mapper, | |
| bazel_test_packages, | |
| build_utils, | |
| stdio_redirection, | |
| thread_pool_helpers, | |
| ) | |
| from build.bazel.scripts.bazel_action_file_copy_utils import ( | |
| FilePath, | |
| check_if_need_to_copy_file, | |
| copy_directory_if_changed, | |
| hardlink_or_copy_writable, | |
| write_file_if_changed, | |
| ) | |
| # LINT.ThenChange( | |
| # //build/bazel/bazel_action.gni:bazel_action_python_script_inputs, | |
| # //build/bazel/scripts/BUILD.gn:bazel_action_impl_imports, | |
| # ) | |
| # Set this to True to debug operations locally in this script. | |
| # IMPORTANT: Setting this to True will result in Ninja timeouts in CQ | |
| # due to the stdout/stderr logs being too large. | |
| _DEBUG = False | |
| # Set this to True to debug the bazel query cache. | |
| _DEBUG_BAZEL_QUERIES = _DEBUG | |
| # Set this to True to debug the export of debug symbols. | |
| _DEBUG_SYMBOL_EXPORT = _DEBUG | |
| def debug(msg: str) -> None: | |
| # Print debug message to stderr if _DEBUG is True. | |
| if _DEBUG: | |
| print("BAZEL_ACTION_DEBUG: " + msg, file=sys.stderr) | |
| # Directory where to find Starlark input files. | |
| _STARLARK_DIR = os.path.join(os.path.dirname(_SCRIPT_DIR), "starlark") | |
| class BazelActionError(Exception): | |
| """A Bazel action error was encountered.""" | |
| class BazelActionScriptError(BazelActionError): | |
| """Errors detected by the Bazel action script, not Bazel itself. | |
| The exception contains a message describing the error, that has | |
| not been printed to stderr yet. | |
| """ | |
| class SourceFilesNotFoundError(BazelActionScriptError): | |
| """No source files found for a target.""" | |
| class FileOutputConfigError(BazelActionScriptError): | |
| """Invalid configuration for file outputs.""" | |
| class DirectoryOutputConfigError(BazelActionScriptError): | |
| """Invalid configuration for directory outputs.""" | |
| class MissingOutputDirectoryError(BazelActionScriptError): | |
| """Missing output directory after build.""" | |
| class MissingTrackedFileError(BazelActionScriptError): | |
| """Missing tracked file inside output directory.""" | |
| class PackageExtractionError(BazelActionScriptError): | |
| """Failed to extract a package archive.""" | |
| @dataclasses.dataclass | |
| class _InputFileGenQueryInfo(object): | |
| """Class for holding info about the genqueries generated for each target.""" | |
| genquery_target_label: str | |
| genquery_output_filename: str | |
| @dataclasses.dataclass | |
| class BazelActionOutputs(object): | |
| """The outputs to copy from Bazel to GN for a given Bazel action.""" | |
| files: list[bazel_action_utils.FileOutput] | |
| directories: list[bazel_action_utils.DirectoryOutput] | |
| packages: list[bazel_action_utils.PackageOutput] | |
| final_symlinks: list[bazel_action_utils.FinalSymlinkOutput] | |
| copy_debug_symbols: bool = False | |
| export_test_packages: bool = False | |
| def __len__(self) -> int: | |
| return ( | |
| len(self.files) | |
| + len(self.directories) | |
| + len(self.packages) | |
| + len(self.final_symlinks) | |
| ) | |
| class BazelDebugSymbolMap(dict[str, list[DebugSymbolEntryType]]): | |
| """A mapping from bazel target labels to lists of DebugSymbolEntryType values.""" | |
| @dataclasses.dataclass | |
| class BazelActionResult(object): | |
| """Wrapper for results of executing a bazel action. | |
| Fields: | |
| configured_args: A list of args usable for bazel queries in the same configuration. | |
| output_files: A list of the output paths that were actually copied to (updated) | |
| by this action. | |
| source_files: A dict of lists of the Bazel build files and input source files | |
| that were used by each of the built targets, as standard filesystem paths, | |
| keyed by the bazel target. | |
| rust_project_crates: A list of CrateSpec dictionaries describing the crates of the | |
| the build targets and their dependencies. | |
| debug_symbols: A BazelDebugSymbolMap of DebugSymbolEntryType dictionaries for each | |
| built target, keyed by the bazel target label. | |
| """ | |
| configured_args: list[str] | |
| output_files: list[Path] | |
| source_files: dict[str, list[str]] | |
| rust_crates: list[bazel_rust_analyzer_utils.Crate] | |
| debug_symbols: BazelDebugSymbolMap = dataclasses.field( | |
| default_factory=BazelDebugSymbolMap | |
| ) | |
| class BazelActionRunner(object): | |
| """Used to run Bazel actions and copy all the outputs back to the Ninja outdir. | |
| This class is configured based on the global context of the build (both static | |
| args set via GN metadata and build-specific environment variables). It can | |
| then be used to run multiple Bazel actions in sequence, with that global | |
| configuration. | |
| See the `run()` method for per-action calling. | |
| """ | |
| def __init__( | |
| self, | |
| bazel_paths: build_utils.BazelPaths, | |
| global_args: bazel_action_utils.BazelGlobalArguments, | |
| query_cache: build_utils.BazelQueryCache, | |
| ) -> None: | |
| self.paths = bazel_paths | |
| self.global_args = global_args | |
| self.launcher = build_utils.BazelLauncher( | |
| bazel_paths.launcher, | |
| log_err=lambda msg: ( | |
| print(f"BAZEL_ACTION_ERROR: {msg}", file=sys.stderr) | |
| if _DEBUG | |
| else None | |
| ), | |
| ) | |
| self.rbe_settings = ( | |
| bazel_action_utils.BazelRbeSettings.create_from_build_dir( | |
| bazel_paths.ninja_build_dir | |
| ) | |
| ) | |
| self.query_cache = query_cache | |
| def run( | |
| self, | |
| command: str, | |
| platform_config: str, | |
| platform_label: str, | |
| targets: list[str], | |
| outputs: BazelActionOutputs, | |
| time_profile: build_utils.TimeProfile, | |
| ) -> BazelActionResult: | |
| """Run a bazel command. | |
| Args: | |
| targets: The list of bazel target labels to build/test/etc. | |
| extra_args: (temporary) list of bazel cli args to append to the command | |
| Returns: | |
| (temporary) A list of debug symbol manifest paths | |
| Raises: | |
| BazelActionError on any failure. | |
| """ | |
| time_profile.start( | |
| "construct_cmd_args", | |
| "Constructing the command-line arguments for Bazel", | |
| ) | |
| # Calculate the platform configuration needed for the build and subsequent queries. | |
| configured_args = calculate_platform_config_args( | |
| platform_config, | |
| platform_label, | |
| self.rbe_settings, | |
| ) | |
| # These are the args that are listed after the command, targets, and configured | |
| # args. | |
| cmd_args = [] | |
| # This returns an empty list of there's nothing for it to set, otherwise it sets all the | |
| # params appropriately based on the number of cores and the ENV vars. | |
| cmd_args += calculate_jobs_params(self.rbe_settings) | |
| if self.global_args.quiet: | |
| cmd_args += ["--config=quiet"] | |
| else: | |
| # Normal builds always need to print INFO level messages so that | |
| # warnings can be emitted from bazel ctx.run_shell() commands, | |
| # otherwise any stderr/stdout warnings from tools are eaten by | |
| # the build. | |
| # | |
| # Known uses: | |
| # - size checker outputs | |
| # - developer overrides warnings for assembly | |
| # | |
| cmd_args += ["--config=verbose"] | |
| cmd_args += [ | |
| # Ensure that all debug symbols are properly generated during the build | |
| # The aspect will also generate extra manifests whose paths will be | |
| # printed to Bazel's stderr DEBUG lines, which will be filtered | |
| # below to retrieve them (and hide the output from the user). | |
| # | |
| # Doing so avoids doing an extra `bazel build` command to get the | |
| # same data. See https://fxbug.dev/452591388 | |
| "--output_groups=+debug_symbol_files", | |
| "--aspects=//build/bazel/debug_symbols:aspects.bzl%generate_manifest", | |
| ] | |
| cmd_args += [ | |
| # Ensure rust-analyzer manifests are generated during the build. | |
| "--output_groups={}".format( | |
| ",".join( | |
| f"+{group}" | |
| for group in bazel_rust_analyzer_utils.RUST_ANALYZER_OUTPUT_GROUPS | |
| ) | |
| ), | |
| "--aspects=//build/bazel/aspects:rust_analyzer.bzl%generate_rust_analyzer_manifest", | |
| ] | |
| cmd_args += [ | |
| # Ensure source file lists are generated during the build to avoid doing one slow | |
| # query for each top-level Bazel target. | |
| "--output_groups=+fuchsia_sources_manifest", | |
| "--aspects=//build/bazel/aspects:source_files.bzl%generate_source_files_manifest", | |
| ] | |
| # Add --sandbox_debug if enabled in the build environment. | |
| if self.global_args.sandbox_debug: | |
| cmd_args += ["--sandbox_debug"] | |
| # Always enable verbose failures | |
| # LINT.IfChange(verbose_failures) | |
| cmd_args += ["--verbose_failures"] | |
| # LINT.ThenChange(/tools/integration/fint/fint_build.py:verbose_failures) | |
| # To capture the set of dependencies for targets, a genquery that can find | |
| # all the BUILD.bazel and .bzl files needed for each target. | |
| # | |
| # The build file must be deleted after each build command. | |
| time_profile.start( | |
| "input_files_genquery", "Generating buildfiles_genquery/BUILD.bazel" | |
| ) | |
| ( | |
| input_files_genqueries, | |
| genquery_build_file, | |
| ) = self._create_buildfiles_genqueries(targets) | |
| # Add the genquery target to the list of targets and add those to the | |
| # command line for bazel. IMPORTANT: The aspects listed above will also apply to | |
| # these targets and generate corresponding files in the buildfiles_genquery/ directory. | |
| cmd_args += targets | |
| cmd_args += [ | |
| genquery.genquery_target_label | |
| for genquery in input_files_genqueries.values() | |
| ] | |
| # Construct the entire command (this is done here, as a single block, for | |
| # clarity on how the various lists above are brought together to make the | |
| # command line that we'll use with Bazel). | |
| cmd: list[str] = [command] | |
| cmd += configured_args | |
| cmd += cmd_args | |
| # Now that there's a complete command string calculated, print it to debug or the command | |
| # file output if we have one. | |
| if _DEBUG: | |
| # This is all arguments on one line, so that they can be run via cut/paste. | |
| debug( | |
| "BUILD_CMD: " | |
| + build_utils.cmd_args_to_string([self.paths.launcher] + cmd) | |
| ) | |
| with contextlib.ExitStack() as on_exit: | |
| # Remove all files in buildfiles_genquery/ after the action completes, to make sure | |
| # that ninja doesn't see them as files that can cause consistency or non-convergence | |
| # issues. | |
| genquery_files_to_cleanup = [] | |
| def _cleanup_genqueries() -> None: | |
| # First, the genquery BUILD.bazel file itself. | |
| try: | |
| genquery_build_file.unlink() | |
| except Exception: | |
| pass | |
| # Then the genquery output files. | |
| for path in genquery_files_to_cleanup: | |
| try: | |
| path.unlink() | |
| except Exception: | |
| pass | |
| try: | |
| (self.paths.workspace / "buildfiles_genquery").rmdir() | |
| except Exception: | |
| pass | |
| on_exit.callback(_cleanup_genqueries) | |
| aspect_manifest_outputs = ( | |
| self._invoke_bazel_and_return_aspect_outputs( | |
| targets, | |
| cmd, | |
| time_profile, | |
| ) | |
| ) | |
| # Parse genquery output files | |
| genquery_output_map = {} | |
| for entry in aspect_manifest_outputs.genquery_output_files: | |
| # Expected entry format <label>,<file_path>, e.g. | |
| # @@//buildfiles_genquery:src_foo_bar.buildfiles.txt,bazel-out/k8-fastbuild/bin/buildfiles_genquery/src_foo_bar.buildfiles.txt | |
| label, comma, path_str = entry.partition(",") | |
| if comma == ",": | |
| path = self.paths.execroot / path_str | |
| genquery_output_map[label] = path | |
| genquery_output_map[ | |
| build_utils.canonicalize_label(label) | |
| ] = path | |
| genquery_output_map[ | |
| build_utils.normalize_label(label) | |
| ] = path | |
| genquery_files_to_cleanup.extend(genquery_output_map.values()) | |
| debug_symbol_manifest_paths = ( | |
| aspect_manifest_outputs.debug_symbol_manifest_paths | |
| ) | |
| rust_manifest_paths = ( | |
| aspect_manifest_outputs.rust_analyzer_manifest_paths | |
| ) | |
| source_files_manifest_paths = ( | |
| aspect_manifest_outputs.source_files_manifest_paths | |
| ) | |
| input_files = {} | |
| if command == "build": | |
| time_profile.start( | |
| "read_genquery_outputs", | |
| "Read the results from the genqueries", | |
| ) | |
| if not self.global_args.quiet: | |
| print( | |
| f"Gathering input paths from {len(targets)} targets for ninja depfile generation..." | |
| ) | |
| input_files = self._parse_buildfiles_genquery_results_and_source_files_manifests( | |
| input_files_genqueries, | |
| genquery_output_map, | |
| source_files_manifest_paths, | |
| time_profile, | |
| ) | |
| if command == "build" and outputs.packages: | |
| # If we have packages, query to get the paths to the package archives, | |
| # as they need to be copied to the Ninja outdir along with any other files. | |
| time_profile.start( | |
| "package_info", | |
| "Run cquery to extract Fuchsia package archiveinformation", | |
| ) | |
| package_archive_files = self.query_for_package_archives( | |
| configured_args, outputs.packages | |
| ) | |
| outputs.files.extend(package_archive_files) | |
| # Now copy all the outputs. This is a separate function to help break up the run() | |
| # functionality for better clarity. | |
| output_copier = _BazelOutputCopier(self.paths) | |
| if not self.global_args.quiet: | |
| print( | |
| f"Inspecting {len(outputs)} outputs from Bazel for copying back to GN/Ninja..." | |
| ) | |
| all_output_files = output_copier.copy(outputs, time_profile) | |
| if command == "build" and outputs.export_test_packages: | |
| # Do this now, while the packages' files are top-level outputs of | |
| # the invocation that just finished. A later invocation could | |
| # rebuild some of them as non-top-level outputs and leave them | |
| # missing from disk, or replant the execroot without the external | |
| # repositories that some of their blobs come from. | |
| time_profile.start( | |
| "export_test_packages", | |
| "Export Bazel test packages to the Ninja build directory", | |
| ) | |
| try: | |
| bazel_test_packages.export_packages( | |
| self.paths, | |
| bazel_test_packages.read_packages_list( | |
| self.paths.ninja_build_dir | |
| ), | |
| ) | |
| except (OSError, KeyError, ValueError) as e: | |
| raise BazelActionError( | |
| f"Failed to export Bazel test packages: {e}" | |
| ) from e | |
| if command == "build" and outputs.packages: | |
| # Extract the package archives that request it, now that they have | |
| # been copied into the Ninja build directory. | |
| all_output_files.extend( | |
| self._extract_package_archives( | |
| outputs.packages, all_output_files, time_profile | |
| ) | |
| ) | |
| # Perform the merging of debug symbols data, and optionally copy | |
| # and write the output to a manifest. | |
| need_to_copy_debug_symbols = outputs.copy_debug_symbols or any( | |
| entry.copy_debug_symbols | |
| for entry in outputs.packages + outputs.directories | |
| ) | |
| debug_symbols = self._handle_debug_symbols( | |
| debug_symbol_manifest_paths, | |
| need_to_copy_debug_symbols, | |
| time_profile, | |
| ) | |
| # Handle the rust analyzer manifests. | |
| crate_spec_paths: set[Path] = set() | |
| for manifest_path in rust_manifest_paths: | |
| manifest_path = self.paths.execroot / manifest_path | |
| with manifest_path.open() as f: | |
| manifest_json = json.load(f) | |
| crate_spec_paths.update( | |
| self.paths.execroot / spec | |
| for spec in manifest_json["crate_specs"] | |
| ) | |
| rust_crate_specs = ( | |
| bazel_rust_analyzer_utils.load_crate_specs_from_json_files( | |
| list(crate_spec_paths), | |
| self.paths, | |
| ) | |
| ) | |
| rust_crates = bazel_rust_analyzer_utils.convert_crate_specs_to_rust_project_crates( | |
| rust_crate_specs | |
| ) | |
| return BazelActionResult( | |
| configured_args=configured_args, | |
| output_files=all_output_files, | |
| source_files=input_files, | |
| rust_crates=rust_crates, | |
| debug_symbols=debug_symbols, | |
| ) | |
| def query_for_package_archives( | |
| self, | |
| configured_args: list[str], | |
| packages: list[bazel_action_utils.PackageOutput], | |
| ) -> list[bazel_action_utils.FileOutput]: | |
| """Given a list of PackageOutputs, query to find all the archive files to copy. | |
| This basically is converting PackageOutput entries into FileOutput entries. | |
| """ | |
| # Query against all package output targets at once. The query results | |
| # embed the target labels so that they can be matched with the ninja | |
| # output paths. | |
| query_result = run_starlark_cquery( | |
| self.query_cache, | |
| self.launcher, | |
| configured_args, | |
| [entry.package_label for entry in packages], | |
| "FuchsiaPackageInfo_archive.cquery", | |
| ) | |
| # The results are lines of json, so split them and parse each. They're | |
| # each a dict of: | |
| # target: "@@<the bazel target>"" | |
| # archive_path: "<path to the archive file>"" | |
| results: dict[str, str] = { | |
| build_utils.normalize_label(p["target"]): p["archive_path"] | |
| for p in [json.loads(line) for line in query_result] | |
| } | |
| # Now match each package output label to its results and add to the file | |
| # copies to perform. | |
| package_archive_paths: list[bazel_action_utils.FileOutput] = [] | |
| for entry in packages: | |
| bazel_archive_path = results.get(entry.package_label) | |
| # This is just in case we don't get a result for a label, or they | |
| # change format on us. | |
| assert bazel_archive_path | |
| package_archive_paths.append( | |
| bazel_action_utils.FileOutput( | |
| bazel_path=str(self.paths.execroot / bazel_archive_path), | |
| ninja_path=entry.archive_path, | |
| ) | |
| ) | |
| return package_archive_paths | |
| def _extract_package_archives( | |
| self, | |
| packages: list[bazel_action_utils.PackageOutput], | |
| updated_files: list[Path], | |
| time_profile: build_utils.TimeProfile, | |
| ) -> list[Path]: | |
| """Extract the package archives of PackageOutputs into package_out_dir. | |
| This replaces the use of a separate `prebuilt_package()` GN target to | |
| extract the archive, which cannot be traced as a hermetic action | |
| because the set of blobs it writes is not known in advance. | |
| To preserve Ninja's `restat` behavior, an archive is only extracted if | |
| it was updated by this action, or if any of the extracted files that | |
| are tracked by Ninja are missing. | |
| Args: | |
| packages: The package outputs of this action. | |
| updated_files: The Ninja output paths updated by this action. | |
| time_profile: The TimeProfile to update. | |
| Returns: | |
| The list of Ninja output paths updated by the extraction. | |
| Raises: | |
| PackageExtractionError if package-tool fails. | |
| """ | |
| updated = set(updated_files) | |
| build_dir = self.paths.ninja_build_dir | |
| def _needs_extraction( | |
| package: bazel_action_utils.PackageOutput, | |
| ) -> bool: | |
| """Returns True if the package archive needs to be extracted. | |
| This is the case if the archive was just updated by this action | |
| (i.e. the Bazel output changed), or if any of the extracted files | |
| that are tracked by Ninja are missing (e.g. a clean build, or a | |
| previous extraction that failed or was interrupted). | |
| Otherwise, the previously extracted content is up to date, and | |
| skipping extraction preserves the timestamps of the tracked files | |
| so that Ninja's `restat` can prune downstream actions. | |
| """ | |
| if Path(package.archive_path) in updated: | |
| return True | |
| return not all( | |
| (build_dir / p).exists() | |
| for p in package.extracted_ninja_paths() | |
| ) | |
| to_extract = [p for p in packages if _needs_extraction(p)] | |
| if not to_extract: | |
| return [] | |
| time_profile.start( | |
| "extract_package_archives", | |
| "Extract Fuchsia package archives into the Ninja build directory", | |
| ) | |
| def _extract(package: bazel_action_utils.PackageOutput) -> str | None: | |
| out_dir = build_dir / package.package_out_dir | |
| # Remove the outputs of previous extractions, to avoid keeping | |
| # stale files around. Removing the files tracked by Ninja first | |
| # also ensures that `_needs_extraction()` returns True on the next | |
| # build if extraction fails or is interrupted. | |
| for p in package.extracted_ninja_paths(): | |
| (build_dir / p).unlink(missing_ok=True) | |
| for subdir in ("blobs", "manifests"): | |
| shutil.rmtree(out_dir / subdir, ignore_errors=True) | |
| cmd = [ | |
| self.global_args.package_tool, | |
| "package", | |
| "archive", | |
| "extract", | |
| package.archive_path, | |
| "-o", | |
| package.package_out_dir, | |
| "--repository", | |
| "fuchsia.com", | |
| "--blobs-json", | |
| ] | |
| ret = subprocess.run( | |
| cmd, cwd=build_dir, capture_output=True, text=True | |
| ) | |
| if ret.returncode == 0: | |
| return None | |
| # The tracked outputs were removed above, so the next build will | |
| # retry extraction even if the Bazel output did not change. | |
| return ( | |
| f"Failed to extract {package.archive_path} for {package.package_label}:\n" | |
| + f" {build_utils.cmd_args_to_string(cmd)}\n" | |
| + ret.stdout | |
| + ret.stderr | |
| ) | |
| errors = [ | |
| e | |
| for e in thread_pool_helpers.map_threaded(_extract, to_extract) | |
| if e | |
| ] | |
| if errors: | |
| raise PackageExtractionError( | |
| "\nBAZEL_ACTION_ERROR: " + "\n".join(errors) | |
| ) | |
| return [ | |
| Path(p) | |
| for package in to_extract | |
| for p in package.extracted_ninja_paths() | |
| ] | |
| def _create_buildfiles_genqueries( | |
| self, targets: list[str] | |
| ) -> tuple[dict[str, _InputFileGenQueryInfo], Path]: | |
| """Create a BUILD.bazel file that defines a genquery that returns all build files used. | |
| Generate a BUILD.bazel file that defines a genquery() target for listing | |
| all build files that we need to include in the depfile for this target, | |
| i.e. any changes in these files should trigger a rebuild of this target. | |
| Args: | |
| targets: The list of Bazel targets to create the query for | |
| Returns: | |
| Tuple of: | |
| - Dict of _InputFileGenQueryInfo objects, keyed by the Bazel target the query is for | |
| - Path to the generated source file | |
| IMPORTANT: These files should be removed after the Bazel build command below | |
| to ensure our consistency builders do not flake on it. Otherwise the content | |
| of $BAZEL_EXECROOT/bazel-out/<config_dir>/bin/buildfiles_genquery/genquery | |
| after a full build will reflect the last bazel_action() command that was | |
| invoked by Ninja, whose scheduling is not deterministic. In certain cases | |
| Ninja may decide between two builds to schedule bazel_action() commands | |
| in a different order, leaving two files with different content in each | |
| build's relative file path, creating a puzzling consistency error. | |
| """ | |
| query_buildfile_lines = [ | |
| "# AUTO-GENERATED - DO NOT EDIT!", | |
| "", | |
| "", | |
| ] | |
| generated_targets = {} | |
| for target in targets: | |
| input_files_target_name = filename_from_target_label( | |
| target, "buildfiles.txt" | |
| ) | |
| query_buildfile_lines += [ | |
| f"# Generated queries for the target: {target}", | |
| f"genquery(", | |
| f' name = "{input_files_target_name}",', | |
| f' expression = "buildfiles(deps({target}))",', | |
| f' scope = ["{target}"],', | |
| f' opts = ["--output=label"],', | |
| f" strict = False,", # See https://fxbug.dev/500609897 | |
| f")", | |
| "", | |
| "", | |
| ] | |
| label = f"@@//buildfiles_genquery:{input_files_target_name}" | |
| generated_targets[target] = _InputFileGenQueryInfo( | |
| label, | |
| input_files_target_name, | |
| ) | |
| genquery_build_content = "\n".join(query_buildfile_lines) | |
| if False: | |
| print( | |
| f"GENQUERY BUILD CONTENT:\n{genquery_build_content}\n", | |
| file=sys.stderr, | |
| ) | |
| genquery_build_file = ( | |
| self.paths.workspace / "buildfiles_genquery/BUILD.bazel" | |
| ) | |
| write_file_if_changed(genquery_build_file, genquery_build_content) | |
| return generated_targets, genquery_build_file | |
| def _parse_buildfiles_genquery_results_and_source_files_manifests( | |
| self, | |
| genqueries: dict[str, _InputFileGenQueryInfo], | |
| genquery_output_map: dict[str, Path], | |
| source_files_manifest_paths: list[str], | |
| time_profile: build_utils.TimeProfile, | |
| ) -> dict[str, list[str]]: | |
| """Parse the output files of the genqueries, then query for each targets source files. | |
| Args: | |
| genqueries: A dictionary mapping target labels to their genquery info. | |
| genquery_output_map: A dictionary mapping genquery target labels to their output file paths. | |
| source_files_manifest_paths: A list of paths to the source files manifests. | |
| time_profile: Time profile to use for timing. | |
| Returns: | |
| A dictionary mapping target labels to their source file paths, relative to | |
| the Ninja build directory. | |
| """ | |
| time_profile.start("parse_genquery_results_threadpooled") | |
| def _parse_labels( | |
| target: str, | |
| info: _InputFileGenQueryInfo, | |
| ) -> tuple[str, list[str]]: | |
| """A worker function for the threadpool to read the genquery output | |
| This reads the output of one query, and maps the labels into file paths. | |
| """ | |
| label = info.genquery_target_label | |
| path = genquery_output_map.get(label) | |
| if not path: | |
| raise BazelActionScriptError( | |
| f"Missing genquery output path for {info.genquery_target_label} in aspect outputs" | |
| ) | |
| return ( | |
| target, | |
| path.read_text().splitlines(), | |
| ) | |
| # Use a thread pool for this to to read and parse all the files in parallel | |
| # For smaller sets of targets, this is slower, but as the number of targets | |
| # scales up it makes the parsing faster. | |
| input_file_labels = dict( | |
| thread_pool_helpers.starmap_threaded( | |
| _parse_labels, genqueries.items() | |
| ) | |
| ) | |
| time_profile.start("map_labels_to_paths") | |
| # now map all the labels into source files. This is rather slow, so | |
| # reuse the same label mapper so that it can cache results that are | |
| # in common between the different targets. | |
| label_mapper = bazel_label_mapper.BazelLabelMapper( | |
| str(self.paths.workspace), str(self.paths.ninja_build_dir) | |
| ) | |
| input_files = dict( | |
| [ | |
| (target, list(label_mapper.get_sources_for_labels(labels))) | |
| for target, labels in input_file_labels.items() | |
| ] | |
| ) | |
| time_profile.start("map_source_paths") | |
| source_mapper = bazel_source_path_mapper.BazelSourcePathMapper( | |
| self.paths | |
| ) | |
| # Read the source files manifests and build a { label -> sources } map. | |
| input_file_paths: dict[str, set[str]] = {} | |
| for manifest_path in source_files_manifest_paths: | |
| with (self.paths.execroot / manifest_path).open() as f: | |
| manifest_json = json.load(f) | |
| # LINT.IfChange(source_files_list_schema) | |
| label = manifest_json["label"] | |
| assert isinstance(label, str) | |
| sources = manifest_json["sources"] | |
| assert isinstance(sources, list) | |
| # LINT.ThenChange(//build/bazel/aspects/source_files.bzl:source_files_list_schema) | |
| assert label.startswith( | |
| "@@//" | |
| ), f"Invalid target label in source file manifest (should start with @@//): {label}" | |
| label = build_utils.normalize_label(label) | |
| # The aspect also walked about out buildfiles_genquery files, ignore them. | |
| if label.startswith("//buildfiles_genquery"): | |
| continue | |
| input_file_paths[label] = set(sources) | |
| # The prefix to prepend to all source paths to make them relative to the ninja outdir | |
| # as expected for the Ninja depfile. | |
| source_prefix = ( | |
| os.path.relpath(self.paths.fuchsia_dir, self.paths.ninja_build_dir) | |
| + "/" | |
| ) | |
| for target, paths in input_file_paths.items(): | |
| sources = [] | |
| for path in paths: | |
| source_path = source_mapper.resolve_path(path) | |
| if source_path: | |
| sources.append(source_prefix + source_path) | |
| # A target named in an extra_bazel_targets_file may be a | |
| # test_suite(), which Bazel expands into its member tests. The | |
| # aspect then reports sources for those members, which were never | |
| # requested by label and so have no entry yet. | |
| input_files.setdefault(target, []).extend(sources) | |
| time_profile.stop() | |
| return input_files | |
| def _invoke_bazel_and_return_aspect_outputs( | |
| self, | |
| targets: list[str], | |
| cmd_args: list[str], | |
| time_profile: build_utils.TimeProfile, | |
| ) -> bazel_action_utils.AspectManifestOutputs: | |
| """A helper function to handle the invocation of Bazel and extract aspect manifests. | |
| Aspect manifests and genquery output paths are discovered directly from the | |
| local Build Event Protocol (BEP) stream. | |
| Args: | |
| targets: List of Bazel target labels to build. | |
| cmd_args: Additional arguments to pass to Bazel. | |
| time_profile: Time profile to use for timing. | |
| Returns: | |
| An AspectManifestOutputs instance. | |
| """ | |
| with tempfile.NamedTemporaryFile( | |
| prefix="bazel_bep_", | |
| suffix=".json", | |
| dir=self.paths.workspace, | |
| delete=False, | |
| ) as tmp_bep: | |
| bep_json_path = Path(tmp_bep.name) | |
| tmp_bep.close() | |
| cmd_args = list(cmd_args) + [ | |
| f"--build_event_json_file={bep_json_path}", | |
| "--output_groups=+debug_symbol_manifest", | |
| ] | |
| time_profile.start("run_bazel_command", "Run the Bazel command.") | |
| if self.global_args.quiet: | |
| # Still capture output to print its content if there is an error | |
| # running the build command below. | |
| output_sink = stdio_redirection.BytesOutputSink() | |
| with stdio_redirection.PipeOutputSink( | |
| output_sink, use_pty=False | |
| ) as pipe_sink: | |
| # This makes mypy happy, as it can't detect the type correctly in the 'with' statement | |
| assert isinstance(pipe_sink, stdio_redirection.PipeOutputSink) | |
| ret = self.launcher.run_bazel_command( | |
| cmd_args, | |
| stdout=pipe_sink.get_write_fd(), | |
| stderr=pipe_sink.get_write_fd(), | |
| ) | |
| else: | |
| # Inherit regular stdout / stderr directly from the parent process. | |
| ret = self.launcher.run_bazel_command( | |
| cmd_args, | |
| stdout=None, | |
| stderr=None, | |
| ) | |
| if ret.returncode != 0: | |
| bep_json_path.unlink(missing_ok=True) | |
| if self.global_args.quiet: | |
| # Print the captured outputs in quiet mode to help debugging build errors. | |
| if output_sink.data: | |
| sys.stderr.buffer.write(output_sink.data) | |
| sys.stderr.flush() | |
| # Detect the error message corresponding to a Bazel target | |
| # referencing a @gn_targets//<dir>:<name> label | |
| # that does not exist. This happens when the GN bazel_action() | |
| # fails to depend on the proper bazel_input_file() or | |
| # bazel_input_directory() dependency. | |
| # | |
| # NOTE: Path to command.log should be stable, because we explicitly set | |
| # output_base. See https://bazel.build/run/scripts#command-log. | |
| verify_error_msg = verify_unknown_gn_targets( | |
| (self.paths.output_base / "command.log") | |
| .read_text() | |
| .splitlines(), | |
| targets, | |
| ) | |
| if verify_error_msg: | |
| raise BazelActionError(verify_error_msg) | |
| # Assume Bazel printed a reasonable error to stderr already. | |
| raise BazelActionError() | |
| try: | |
| time_profile.start( | |
| "parse_bep_manifests", "Parse manifest files from BEP events." | |
| ) | |
| return bazel_action_utils.parse_build_event_manifests( | |
| bep_json_path, self.paths.execroot | |
| ) | |
| finally: | |
| bep_json_path.unlink(missing_ok=True) | |
| def _handle_debug_symbols( | |
| self, | |
| bazel_manifest_paths: list[str], | |
| perform_copy: bool, | |
| time_profile: build_utils.TimeProfile, | |
| ) -> BazelDebugSymbolMap: | |
| """Perform any post-build operations that need to happen with the debug symbols. | |
| If they need to be copied, then do so. | |
| """ | |
| if not bazel_manifest_paths: | |
| return BazelDebugSymbolMap() | |
| ( | |
| debug_symbols_manifest, | |
| debug_symbols_by_target, | |
| ) = merge_debug_symbol_manifests( | |
| bazel_manifest_paths, | |
| bazel_execroot=self.paths.execroot, | |
| build_dir=self.paths.ninja_build_dir, | |
| time_profile=time_profile, | |
| prebuilt_build_id_dir=self.paths.fuchsia_dir | |
| / "prebuilt" | |
| / ".build-id", | |
| ) | |
| if perform_copy and debug_symbols_manifest: | |
| time_profile.start( | |
| "copy_debug_symbols", | |
| "Copy debug symbols to Ninja build directory.", | |
| ) | |
| copy_debug_symbols_to_build_dir( | |
| self.paths.ninja_build_dir, debug_symbols_manifest | |
| ) | |
| return debug_symbols_by_target | |
| def calculate_platform_config_args( | |
| platform_config: str, | |
| platform_label: str, | |
| rbe_settings: bazel_action_utils.BazelRbeSettings, | |
| ) -> list[str]: | |
| """Constructs the CLI args that define the 'common platform configuration' as needed by the command and any subsequent queries. | |
| This is an internal helper that creates the cli args, including the RBE | |
| settings, so that any post-build queries can use the same platform config | |
| as the action itself did. | |
| Args: | |
| platform_config: The name of the bazel config to enable for the platform | |
| platform_label: The bazel label of the platform | |
| rbe_settings: The global RBE settings for the build | |
| Returns: | |
| List of strings that are the CLI args that together configure Bazel | |
| correctly for the build and for any subsequent queries. | |
| """ | |
| platform_config_args = [ | |
| f"--config={platform_config}", | |
| f"--platforms={platform_label}", | |
| ] | |
| # When remote builds are enabled, append the right build arguments. | |
| # These must appear on the Bazel command-line otherwise remote builds | |
| # will fail on infra (the reason being that the Bazel wrapper script | |
| # detects these options to add infra-specific proxy configuration | |
| # the the final command-line). | |
| # | |
| # RBE settings are included in the platform config so that the remote cache can be | |
| # utilized for queries. | |
| if rbe_settings.enabled: | |
| assert rbe_settings.exec_strategy != None | |
| # If RBE is enabled, append the chosen RBE config to | |
| # the command line. | |
| platform_config_args += [f"--config={rbe_settings.exec_strategy}"] | |
| return platform_config_args | |
| def calculate_jobs_params( | |
| rbe_settings: bazel_action_utils.BazelRbeSettings, | |
| ) -> list[str]: | |
| """Given the RBE settings and the environment vars, determine what --jobs related params to use, if any.""" | |
| # When running jobs remotely, increase the number of allowed jobs to 10x | |
| # when running jobs locally. This is different from the reclient config | |
| # because this controls the _running_ of jobs, not the checking of the | |
| # cache for jobs. | |
| if rbe_settings.enabled: | |
| cpus = os.cpu_count() | |
| if cpus: | |
| return [ | |
| # Total number of jobs (remote + local) at once | |
| f"--jobs={10 * cpus}", | |
| # The limit on the number of locally-running jobs. | |
| "--local_resources=cpu=HOST_CPUS", | |
| ] | |
| # If an explicit job count was passed to `fx build`, tell Bazel to respect it. | |
| # See https://fxbug.dev/351623259 | |
| job_count = os.environ.get("FUCHSIA_BAZEL_JOB_COUNT") | |
| if job_count: | |
| jobs = int(job_count) | |
| return [f"--jobs={jobs}"] | |
| return [] | |
| def verify_unknown_gn_targets( | |
| build_files_error: list[str], | |
| bazel_targets: list[str], | |
| ) -> str: | |
| """Check for unknown @gn_targets// dependencies. | |
| Args: | |
| build_files_error: list of error lines from bazel build or query. | |
| bazel_targets: list of Bazel targets invoked by the GN bazel_action() target. | |
| Returns: | |
| On success, and empty string. On failure, an human friendly | |
| error message explaining the situation. | |
| """ | |
| missing_ninja_outputs = set() | |
| missing_ninja_packages = set() | |
| for error_line in build_files_error: | |
| if not ("ERROR: " in error_line and "@gn_targets" in error_line): | |
| continue | |
| pos = error_line.find("@@gn_targets") | |
| if pos < 0: | |
| # Should not happen, do not assert and let the caller print the full error | |
| # after this. | |
| return f"UNSUPPORTED ERROR LINE: {error_line}" | |
| ending_pos = error_line.find("'", pos) | |
| if ending_pos < 0: | |
| return f"UNSUPPORTED ERROR LINE: {error_line}" | |
| label = error_line[pos + 1 : ending_pos] # skip first @. | |
| if error_line[:pos].endswith(": no such package '"): | |
| # The line looks like the following when the BUILD.bazel references a label | |
| # that does not belong to a @gn_targets package. | |
| # | |
| # ERROR: <abspath>/BUILD.bazel:<line>:<column>: no such package '@@gn_targets//<dir>': ... | |
| # | |
| # This happens when the GN bazel_action() targets fails to dependon the corresponding | |
| # bazel_input_file() or bazel_input_directory() target, and that none of its other | |
| # dependencies expose other targets from the same package / directory. The error message | |
| # does not give any information about the target name being evaluated by the query though. | |
| missing_ninja_packages.add(label) | |
| elif error_line[:pos].endswith(": no such target '"): | |
| # The line looks like this when the BUILD.bazel files references the wrong | |
| # label from a package exposed in @gn_targets//: | |
| # ERROR: <abspath>/BUILD.bazel:<line>:<column>: no such target '@@gn_targets//<dir>:<name>' ... | |
| missing_ninja_outputs.add(label) | |
| if not missing_ninja_outputs and not missing_ninja_packages: | |
| return "" | |
| missing_outputs = sorted(missing_ninja_outputs) | |
| missing_packages = sorted(missing_ninja_packages) | |
| _ERROR = """ | |
| BAZEL_ACTION_ERROR: Unknown @gn_targets targets. | |
| The following Bazel target(s): {bazel_targets} | |
| """.format( | |
| bazel_targets=" ".join(bazel_targets), | |
| ) | |
| if not missing_packages: | |
| _ERROR += """Which reference the following unknown @gn_targets labels: | |
| {missing_bazel_labels} | |
| To fix this, ensure that bazel_input_file() or bazel_input_directory() | |
| targets are defined in the GN graph for: | |
| {missing_gn_labels} | |
| Then ensure that the GN target depends on them transitively. | |
| """.format( | |
| missing_bazel_labels="\n ".join(missing_outputs), | |
| missing_gn_labels="\n ".join( | |
| f"//{o.removeprefix('@gn_targets//')}" for o in missing_outputs | |
| ), | |
| ) | |
| else: | |
| missing_labels = missing_outputs + missing_packages | |
| missing_build_files = set() | |
| for label in missing_labels: | |
| label = label.removeprefix("@gn_targets//") | |
| gn_dir, sep, gn_name = label.partition(":") | |
| if sep != ":": | |
| gn_dir = label | |
| missing_build_files.add(f"//{gn_dir}/BUILD.gn") | |
| _ERROR += """Which reference the following unknown @gn_targets labels or packages: | |
| {missing_bazel_labels} | |
| To fix this, ensure that bazel_input_file() or bazel_input_directory() | |
| targets are defined in the following build files: | |
| {missing_build_files} | |
| Then ensure that the GN target depends on them transitively. | |
| """.format( | |
| missing_bazel_labels="\n ".join( | |
| missing_outputs + missing_packages | |
| ), | |
| missing_build_files="\n ".join(sorted(missing_build_files)), | |
| ) | |
| return _ERROR | |
| def merge_target_info_outputs( | |
| target_infos: T.Sequence[bazel_action_utils.BazelTargetInfo], | |
| ) -> tuple[BazelActionOutputs, list[Path]]: | |
| """Merge all the outputs and needed input gn targets for the given list of BazelTargetInfos. | |
| Args: | |
| Sequence of BazelTargetInfo objects | |
| Returns: | |
| BazelActionOutputs that merges all target_info's outputs together | |
| List of manifests to GN targets needed by the BazelTargetInfos | |
| """ | |
| # Merge all the expected outputs together | |
| file_outputs: list[bazel_action_utils.FileOutput] = [] | |
| directory_outputs: list[bazel_action_utils.DirectoryOutput] = [] | |
| package_outputs: list[bazel_action_utils.PackageOutput] = [] | |
| final_symlink_outputs: list[bazel_action_utils.FinalSymlinkOutput] = [] | |
| copy_debug_symbols = False | |
| export_test_packages = False | |
| # This is the set of gn input targets manifests | |
| gn_target_manifests: set[Path] = set() | |
| for target_info in target_infos: | |
| file_outputs.extend(target_info.copy_outputs) | |
| directory_outputs.extend(target_info.directory_outputs) | |
| package_outputs.extend(target_info.package_outputs) | |
| final_symlink_outputs.extend(target_info.final_symlink_outputs) | |
| copy_debug_symbols |= target_info.copy_debug_symbols | |
| export_test_packages |= target_info.export_test_packages | |
| gn_target_manifests.add(Path(target_info.gn_targets_manifest)) | |
| return ( | |
| BazelActionOutputs( | |
| file_outputs, | |
| directory_outputs, | |
| package_outputs, | |
| final_symlink_outputs, | |
| copy_debug_symbols, | |
| export_test_packages, | |
| ), | |
| list(gn_target_manifests), | |
| ) | |
| def run_starlark_cquery( | |
| query_cache: build_utils.BazelQueryCache, | |
| bazel_launcher: build_utils.BazelLauncher, | |
| configured_args: list[str], | |
| query_targets: list[str], | |
| starlark_filename: str, | |
| ) -> list[str]: | |
| """Run a Bazel cquery and process its output with a starlark file. | |
| Args: | |
| query_targets: A list of Bazel targets to run the query over. | |
| starlark_filename: Name of starlark file from //build/bazel/starlark. | |
| Returns: | |
| A list of output lines. | |
| Raises: | |
| AssertionError in case of failure. | |
| """ | |
| result = run_bazel_query( | |
| query_cache, | |
| bazel_launcher, | |
| "cquery", | |
| [ | |
| "--config=quiet", | |
| "--output=starlark", | |
| "--starlark:file", | |
| get_input_starlark_file_path(starlark_filename), | |
| ] | |
| + configured_args | |
| + ["set(%s)" % " ".join(query_targets)], | |
| ) | |
| assert result is not None | |
| return result | |
| def get_input_starlark_file_path(filename: FilePath) -> str: | |
| """Return the path of a input starlark file for Bazel queries. | |
| Args: | |
| filename: File name, searched in //build/bazel/starlark/ | |
| Returns: | |
| file path to the corresponding file. | |
| """ | |
| result = os.path.join(_STARLARK_DIR, filename) | |
| assert os.path.isfile(result), f"Missing starlark input file: {result}" | |
| return result | |
| def run_bazel_query( | |
| query_cache: build_utils.BazelQueryCache, | |
| bazel_launcher: build_utils.BazelLauncher, | |
| query_cmd: str, | |
| query_args: list[str], | |
| ) -> list[str] | None: | |
| """Run a Bazel query, return output as list of lines. | |
| Args: | |
| query_cmd: Query command ("query", "cquery" or "aquery"). | |
| query_args: Query arguments. | |
| Returns: | |
| On success, a list of output lines. On failure return None. | |
| """ | |
| return query_cache.get_query_output( | |
| query_cmd, | |
| query_args, | |
| bazel_launcher, | |
| log=lambda m: ( | |
| print(f"DEBUG: {m}", file=sys.stderr) | |
| if _DEBUG_BAZEL_QUERIES | |
| else None | |
| ), | |
| ) | |
| class _BazelOutputCopier(object): | |
| """This exists to consolidate together all of the Bazel output-copying logic.""" | |
| def __init__(self, paths: build_utils.BazelPaths): | |
| self.paths = paths | |
| def copy( | |
| self, | |
| outputs: BazelActionOutputs, | |
| time_profile: build_utils.TimeProfile, | |
| ) -> list[Path]: | |
| """Copy all the given outputs. | |
| Returns a list of all output destination files that were copied to, including tracked | |
| files in DirectoryOutputs. | |
| """ | |
| output_files = self._copy_files(outputs.files, time_profile) | |
| tracked_files = self._copy_directories( | |
| outputs.directories, time_profile | |
| ) | |
| self._make_final_symlinks(outputs.final_symlinks, time_profile) | |
| return output_files + tracked_files | |
| def _copy_files( | |
| self, | |
| file_outputs: list[bazel_action_utils.FileOutput], | |
| time_profile: build_utils.TimeProfile, | |
| ) -> list[Path]: | |
| """Perform all of the file-copying logic. | |
| Returns a list of the destination file paths that were copied to. | |
| """ | |
| time_profile.start( | |
| "check_outputs_for_copying", | |
| "Validate output files to copy are actually files.", | |
| ) | |
| file_copies: list[tuple[Path, Path]] = [] | |
| unwanted_dirs = [] | |
| for file_output in file_outputs: | |
| src_path = self.paths.workspace / file_output.bazel_path | |
| if src_path.is_dir(): | |
| unwanted_dirs.append(src_path) | |
| continue | |
| dst_path = Path(file_output.ninja_path) | |
| file_copies.append((src_path, dst_path)) | |
| if unwanted_dirs: | |
| raise FileOutputConfigError( | |
| "\nDirectories are not allowed in --file-outputs Bazel paths, got directories:\n\n%s\n" | |
| % "\n".join(str(d) for d in unwanted_dirs) | |
| ) | |
| files_to_copy: list[tuple[Path, Path]] = [] | |
| if file_copies: | |
| time_profile.start( | |
| "check_copy_files", | |
| "Check to see if files need to be copied or not.", | |
| ) | |
| files_to_copy = [ | |
| file_copy | |
| for file_copy in thread_pool_helpers.filter_threaded( | |
| check_if_need_to_copy_file, file_copies | |
| ) | |
| if file_copy | |
| ] | |
| if files_to_copy: | |
| time_profile.start( | |
| "copy_files", | |
| "Copy Bazel output files to Ninja build directory", | |
| ) | |
| thread_pool_helpers.starmap_threaded( | |
| hardlink_or_copy_writable, | |
| [ | |
| (src, dst, self.paths.output_base) | |
| for src, dst in files_to_copy | |
| ], | |
| ) | |
| return [dst for _, dst in files_to_copy] | |
| def _copy_directories( | |
| self, | |
| directory_outputs: list[bazel_action_utils.DirectoryOutput], | |
| time_profile: build_utils.TimeProfile, | |
| ) -> list[Path]: | |
| time_profile.start( | |
| "check_output_directories", | |
| "Validate that output directories are ready to be copied.", | |
| ) | |
| dir_copies: list[tuple[Path, Path, list[FilePath]]] = [] | |
| missing_directories: list[Path] = [] | |
| unwanted_files: list[Path] = [] | |
| invalid_tracked_files: list[Path] = [] | |
| for dir_output in directory_outputs: | |
| src_path = self.paths.workspace / dir_output.bazel_path | |
| if not src_path.exists(): | |
| missing_directories.append(src_path) | |
| continue | |
| if not src_path.is_dir(): | |
| unwanted_files.append(src_path) | |
| continue | |
| for tracked_file in dir_output.tracked_files: | |
| tracked_file = src_path / tracked_file | |
| if not tracked_file.is_file(): | |
| invalid_tracked_files.append(tracked_file) | |
| dst_path = Path(dir_output.ninja_path) | |
| dir_copies.append( | |
| ( | |
| src_path, | |
| dst_path, | |
| [Path(f) for f in dir_output.tracked_files], | |
| ) | |
| ) | |
| if missing_directories: | |
| missing_list = "\n ".join(str(d) for d in missing_directories) | |
| error_msg = f""" | |
| BAZEL_ACTION_ERROR: Missing output directory. | |
| The following expected directory output(s) were missing: | |
| {missing_list} | |
| """ | |
| raise MissingOutputDirectoryError(error_msg) | |
| if unwanted_files: | |
| unwanted_list = "\n ".join(str(f) for f in unwanted_files) | |
| error_msg = f""" | |
| BAZEL_ACTION_ERROR: Non-directory output path. | |
| The following output directory path(s) exist but are not directories: | |
| {unwanted_list} | |
| """ | |
| raise DirectoryOutputConfigError(error_msg) | |
| if invalid_tracked_files: | |
| missing_list = "\n ".join(str(f) for f in invalid_tracked_files) | |
| error_msg = f""" | |
| BAZEL_ACTION_ERROR: Missing tracked output file(s). | |
| The following expected tracked file(s) inside output directories were missing or not regular files: | |
| {missing_list} | |
| To fix this, check that the Bazel action or assembly configuration produced the expected files with the expected filenames. | |
| """ | |
| raise MissingTrackedFileError(error_msg) | |
| copied_tracked_files: list[Path] = [] | |
| if dir_copies: | |
| time_profile.start( | |
| "copy_directories", | |
| "Copy Bazel output directories to Ninja build directory", | |
| ) | |
| for src_path, dst_path, tracked_files in dir_copies: | |
| if copy_directory_if_changed(src_path, dst_path, tracked_files): | |
| copied_tracked_files.extend( | |
| [dst_path / file for file in tracked_files] | |
| ) | |
| return copied_tracked_files | |
| def _make_final_symlinks( | |
| self, | |
| final_symlink_outputs: list[bazel_action_utils.FinalSymlinkOutput], | |
| time_profile: build_utils.TimeProfile, | |
| ) -> None: | |
| time_profile.start("symlink_outputs", "Symlink output files.") | |
| final_symlinks: list[tuple[Path, Path]] = [] | |
| for final_symlink_output in final_symlink_outputs: | |
| src_path = self.paths.workspace / final_symlink_output.bazel_path | |
| target_path = src_path.resolve() | |
| link_path = Path(final_symlink_output.ninja_path) | |
| final_symlinks.append((target_path, link_path)) | |
| if final_symlinks: | |
| # This doesn't need to use a thread pool because as of today there's only ever | |
| # one file, in one action, that uses this codepath. | |
| for target_path, link_path in final_symlinks: | |
| build_utils.force_symlink(link_path, target_path) | |
| def filename_from_target_label( | |
| target: str, extension: str | None = None | |
| ) -> str: | |
| """Convert a target label into string that can be a filename. | |
| This isn't a reversible function, as the resultant filename can be | |
| converted to multiple different targets. | |
| TODO: consider using a hash instead to ensure that there are not possible | |
| name collisions | |
| Examples: | |
| > filename_from_target_label("//foo/bar:baz") | |
| "foo_bar_baz" | |
| > filename_from_target_label("//foo/bar:baz", "txt) | |
| "foo_bar_baz.txt" | |
| Args: | |
| target - The bazel target to flatten into a filename. | |
| extension - An optional extension to add to the filename. | |
| """ | |
| temp = target.removeprefix("//").replace("/", "_").replace(":", "_") | |
| if extension: | |
| return temp + "." + extension | |
| else: | |
| return temp | |
| class _PrebuiltPackageExpander(object): | |
| """Expands prebuilt package manifests into debug symbol entries and caches the results.""" | |
| def __init__( | |
| self, | |
| base_path: Path, | |
| prebuilt_build_id_dir: Path, | |
| build_dir: Path, | |
| parser: DebugSymbolsManifestParser, | |
| ): | |
| self.base_path = Path(base_path) | |
| self.prebuilt_build_id_dir = Path(prebuilt_build_id_dir).resolve() | |
| self.build_dir = Path(build_dir).resolve() | |
| self.parser = parser | |
| self.cache: dict[str, list[DebugSymbolEntryType]] = {} | |
| self._extra_input_files: set[Path] = set() | |
| @property | |
| def extra_input_files(self) -> set[Path]: | |
| """Set of implicit input files read during expansion.""" | |
| return self._extra_input_files | |
| def expand( | |
| self, input_entry: dict[str, T.Any] | |
| ) -> list[DebugSymbolEntryType]: | |
| """Expand a prebuilt package entry into a list of debug symbol entries.""" | |
| pkg_manifest_rel = input_entry["package_manifest"] | |
| if pkg_manifest_rel in self.cache: | |
| return self.cache[pkg_manifest_rel] | |
| expanded: list[DebugSymbolEntryType] = [] | |
| pkg_manifest_path = self.base_path / pkg_manifest_rel | |
| if pkg_manifest_path.exists(): | |
| self._extra_input_files.add(pkg_manifest_path) | |
| self.parser.extra_input_files.add(pkg_manifest_path) | |
| with open(pkg_manifest_path, "rt") as pf: | |
| pkg_manifest = json.load(pf) | |
| sources_relative = pkg_manifest.get("blob_sources_relative") | |
| if not sources_relative: | |
| sources_relative = pkg_manifest.get( | |
| "sources_relative", "working_dir" | |
| ) | |
| pkg_manifest_dir = pkg_manifest_path.parent | |
| for blob in pkg_manifest.get("blobs", []): | |
| dest_path = blob["path"] | |
| if dest_path.startswith("meta/"): | |
| continue | |
| blob_source_path = blob["source_path"] | |
| if sources_relative == "file": | |
| blob_path = pkg_manifest_dir / blob_source_path | |
| else: | |
| blob_path = self.base_path / blob_source_path | |
| self._extra_input_files.add(blob_path) | |
| self.parser.extra_input_files.add(blob_path) | |
| build_id = self.parser.extract_build_id(blob_path) | |
| if not build_id or len(build_id) <= 2: | |
| continue | |
| debug_file_path = ( | |
| self.prebuilt_build_id_dir | |
| / build_id[:2] | |
| / f"{build_id[2:]}.debug" | |
| ) | |
| if not debug_file_path.exists(): | |
| continue | |
| self._extra_input_files.add(debug_file_path) | |
| self.parser.extra_input_files.add(debug_file_path) | |
| expanded.append( | |
| { | |
| "debug": os.path.relpath( | |
| debug_file_path, self.build_dir | |
| ), | |
| "elf_build_id": build_id, | |
| "dest_path": dest_path, | |
| "os": input_entry["os"], | |
| "cpu": input_entry["cpu"], | |
| "label": input_entry["label"], | |
| } | |
| ) | |
| self.cache[pkg_manifest_rel] = expanded | |
| return expanded | |
| def expand_container( | |
| self, input_entry: dict[str, T.Any] | |
| ) -> list[DebugSymbolEntryType]: | |
| """Expand an assembly container entry into a list of debug symbol entries.""" | |
| container_rel = input_entry["container_dir"] | |
| if container_rel in self.cache: | |
| return self.cache[container_rel] | |
| expanded: list[DebugSymbolEntryType] = [] | |
| container_path = self.base_path / container_rel | |
| if not container_path.exists(): | |
| self.cache[container_rel] = expanded | |
| return expanded | |
| self._extra_input_files.add(container_path) | |
| self.parser.extra_input_files.add(container_path) | |
| seen_build_ids: set[str] = set() | |
| def _process_blob(blob_path: Path, dest_path: str) -> None: | |
| if not blob_path.exists() or not blob_path.is_file(): | |
| return | |
| self._extra_input_files.add(blob_path) | |
| self.parser.extra_input_files.add(blob_path) | |
| build_id = self.parser.extract_build_id(blob_path) | |
| if not build_id or len(build_id) <= 2 or build_id in seen_build_ids: | |
| return | |
| debug_file_path = ( | |
| self.prebuilt_build_id_dir | |
| / build_id[:2] | |
| / f"{build_id[2:]}.debug" | |
| ) | |
| if not debug_file_path.exists(): | |
| return | |
| seen_build_ids.add(build_id) | |
| self._extra_input_files.add(debug_file_path) | |
| self.parser.extra_input_files.add(debug_file_path) | |
| expanded.append( | |
| { | |
| "debug": os.path.relpath(debug_file_path, self.build_dir), | |
| "elf_build_id": build_id, | |
| "dest_path": dest_path, | |
| "os": input_entry["os"], | |
| "cpu": input_entry["cpu"], | |
| "label": input_entry["label"], | |
| } | |
| ) | |
| def _process_package_manifest(pkg_file: Path) -> None: | |
| if not pkg_file.exists() or not pkg_file.is_file(): | |
| return | |
| try: | |
| with open(pkg_file, "rt") as pf: | |
| pkg_manifest = json.load(pf) | |
| except Exception: | |
| return | |
| if ( | |
| not isinstance(pkg_manifest, dict) | |
| or "blobs" not in pkg_manifest | |
| ): | |
| return | |
| self._extra_input_files.add(pkg_file) | |
| self.parser.extra_input_files.add(pkg_file) | |
| sources_relative = pkg_manifest.get("blob_sources_relative") | |
| if not sources_relative: | |
| sources_relative = pkg_manifest.get( | |
| "sources_relative", "working_dir" | |
| ) | |
| pkg_manifest_dir = pkg_file.parent | |
| for blob in pkg_manifest.get("blobs", []): | |
| dest_path = blob.get("path", "") | |
| if dest_path.startswith("meta/"): | |
| continue | |
| blob_source_path = blob.get("source_path", "") | |
| merkle = blob.get("merkle", "") | |
| blob_path = None | |
| if sources_relative == "file" and blob_source_path: | |
| p = pkg_manifest_dir / blob_source_path | |
| if p.exists(): | |
| blob_path = p | |
| elif blob_source_path: | |
| p = self.base_path / blob_source_path | |
| if p.exists(): | |
| blob_path = p | |
| if blob_path is None and merkle: | |
| # Fall back to container's blobs directory by merkle root | |
| p = container_path / "blobs" / merkle | |
| if p.exists(): | |
| blob_path = p | |
| if blob_path is None and blob_source_path: | |
| # Fall back to container's blobs directory by filename | |
| p = container_path / "blobs" / Path(blob_source_path).name | |
| if p.exists(): | |
| blob_path = p | |
| if blob_path is not None: | |
| _process_blob(blob_path, dest_path) | |
| # 1. Check for package manifests under packages/ and subpackages/ | |
| for sub_dir_name in ("packages", "subpackages"): | |
| sub_dir = container_path / sub_dir_name | |
| if sub_dir.is_dir(): | |
| for pkg_file in sub_dir.rglob("*"): | |
| if pkg_file.is_file(): | |
| _process_package_manifest(pkg_file) | |
| # 2. Check for product_configuration.json | |
| prod_config_file = container_path / "product_configuration.json" | |
| if prod_config_file.is_file(): | |
| self._extra_input_files.add(prod_config_file) | |
| self.parser.extra_input_files.add(prod_config_file) | |
| try: | |
| with open(prod_config_file, "rt") as f: | |
| prod_config = json.load(f) | |
| pkgs_section = prod_config.get("packages", {}) | |
| for pkg_list in ( | |
| pkgs_section.get("base", []), | |
| pkgs_section.get("cache", []), | |
| pkgs_section.get("bootfs", []), | |
| ): | |
| items = ( | |
| pkg_list.values() | |
| if isinstance(pkg_list, dict) | |
| else pkg_list | |
| ) | |
| for entry in items: | |
| manifest_path = ( | |
| entry | |
| if isinstance(entry, str) | |
| else entry.get("manifest") | |
| if isinstance(entry, dict) | |
| else None | |
| ) | |
| if manifest_path: | |
| for base in (container_path, self.base_path): | |
| p = base / manifest_path | |
| if p.is_file(): | |
| _process_package_manifest(p) | |
| break | |
| except Exception: | |
| pass | |
| # 3. Check for image_assembly.json | |
| image_assembly_file = container_path / "image_assembly.json" | |
| if image_assembly_file.is_file(): | |
| self._extra_input_files.add(image_assembly_file) | |
| self.parser.extra_input_files.add(image_assembly_file) | |
| try: | |
| with open(image_assembly_file, "rt") as f: | |
| image_assembly = json.load(f) | |
| for pkg_list in ( | |
| image_assembly.get("base", []), | |
| image_assembly.get("cache", []), | |
| image_assembly.get("bootfs_packages", []), | |
| ): | |
| items = ( | |
| pkg_list.values() | |
| if isinstance(pkg_list, dict) | |
| else pkg_list | |
| ) | |
| for entry in items: | |
| manifest_path = ( | |
| entry | |
| if isinstance(entry, str) | |
| else entry.get("manifest") | |
| if isinstance(entry, dict) | |
| else None | |
| ) | |
| if manifest_path: | |
| for base in (container_path, self.base_path): | |
| p = base / manifest_path | |
| if p.is_file(): | |
| _process_package_manifest(p) | |
| break | |
| for bootfs_file in image_assembly.get("bootfs_files", []): | |
| if isinstance(bootfs_file, dict): | |
| source = bootfs_file.get("source") | |
| destination = bootfs_file.get( | |
| "destination", "bootfs_file" | |
| ) | |
| if source: | |
| for base in (container_path, self.base_path): | |
| p = base / source | |
| if p.is_file(): | |
| _process_blob(p, destination) | |
| break | |
| except Exception: | |
| pass | |
| # 4. Check any remaining blobs directly under blobs/ | |
| blobs_dir = container_path / "blobs" | |
| if blobs_dir.is_dir(): | |
| for blob_file in blobs_dir.iterdir(): | |
| if blob_file.is_file(): | |
| _process_blob(blob_file, f"blobs/{blob_file.name}") | |
| self.cache[container_rel] = expanded | |
| return expanded | |
| def merge_debug_symbol_manifests( | |
| debug_symbol_manifest_paths: list[str], | |
| bazel_execroot: Path, | |
| build_dir: Path, | |
| prebuilt_build_id_dir: Path, | |
| time_profile: build_utils.TimeProfile, | |
| ) -> tuple[list[DebugSymbolEntryType], BazelDebugSymbolMap]: | |
| """Generate final debug symbol manifest and per-target debug symbol map. | |
| Args: | |
| debug_symbol_manifest_paths: The list of "<label>,<manifest_path>" | |
| entries generated by the debug symbol aspect, which were extracted | |
| from the Bazel stderr's DEBUG lines. | |
| bazel_execroot: Path to Bazel execroot. | |
| build_dir: Path to Ninja build directory. | |
| prebuilt_build_id_dir: Path to //prebuilt/.build-id directory. | |
| time_profile: A TimeProfile instance. | |
| Returns: | |
| A tuple of: | |
| - a list of dictionaries describing all unique debug symbols according | |
| to the GN //:debug_symbols schema. | |
| - a BazelDebugSymbolMap mapping each Bazel target label ("//pkg:target") | |
| to its list of debug symbol dictionaries. | |
| """ | |
| time_profile.start( | |
| "merge_debug_symbol_manifests", | |
| "Merge target-specific manifests into final version.", | |
| ) | |
| output_manifest: list[DebugSymbolEntryType] = [] | |
| entries_by_src_debug: dict[str, DebugSymbolEntryType] = {} | |
| src_debugs_by_target: dict[str, list[str]] = {} | |
| parser = DebugSymbolsManifestParser(build_dir) | |
| prebuilt_expander = _PrebuiltPackageExpander( | |
| bazel_execroot, prebuilt_build_id_dir, build_dir, parser | |
| ) | |
| for recorded_entry in debug_symbol_manifest_paths: | |
| # Expected format: "<label>,<manifest_path>", e.g. | |
| # "@@//src/foo:bar,bazel-out/k8-fastbuild/bin/src/foo/bar.debug_symbols.json" | |
| label, comma, manifest_path = recorded_entry.partition(",") | |
| assert ( | |
| comma | |
| ), f"Invalid debug symbols manifest path format (missing comma): {recorded_entry}" | |
| assert label.startswith( | |
| "@@//" | |
| ), f"Invalid target label in debug symbols manifest path (should start with @@//): {label}" | |
| target_label = label.removeprefix("@@") | |
| input_manifest_path = os.path.join(bazel_execroot, manifest_path) | |
| with open(input_manifest_path, "rt") as f: | |
| input_manifest = json.load(f) | |
| target_src_debugs: list[str] = [] | |
| seen_in_target: set[str] = set() | |
| for input_entry in input_manifest: | |
| if "package_manifest" in input_entry: | |
| expanded_entries = prebuilt_expander.expand(input_entry) | |
| elif "container_dir" in input_entry: | |
| expanded_entries = prebuilt_expander.expand_container( | |
| input_entry | |
| ) | |
| else: | |
| expanded_entries = None | |
| if expanded_entries is not None: | |
| for expanded_entry in expanded_entries: | |
| src_debug = expanded_entry["debug"] | |
| if src_debug not in seen_in_target: | |
| seen_in_target.add(src_debug) | |
| target_src_debugs.append(src_debug) | |
| if src_debug not in entries_by_src_debug: | |
| entry = expanded_entry.copy() | |
| entries_by_src_debug[src_debug] = entry | |
| output_manifest.append(entry) | |
| continue | |
| src_debug = input_entry["debug"] | |
| if src_debug not in seen_in_target: | |
| seen_in_target.add(src_debug) | |
| target_src_debugs.append(src_debug) | |
| existing = entries_by_src_debug.get(src_debug) | |
| if existing is not None: | |
| if "dest_path" in input_entry and "dest_path" not in existing: | |
| existing["dest_path"] = input_entry["dest_path"] | |
| continue | |
| # Adjust paths | |
| entry = input_entry.copy() | |
| for key in ("debug", "stripped", "breakpad", "elf_build_id"): | |
| src_path = entry.get(key, "") | |
| if src_path: | |
| # Convert execroot-relative paths to a build-dir relative ones, using | |
| # os.path.realpath() to resolve symlinks properly. | |
| entry[key] = os.path.relpath( | |
| os.path.realpath( | |
| os.path.join(bazel_execroot, src_path) | |
| ), | |
| build_dir, | |
| ) | |
| entries_by_src_debug[src_debug] = entry | |
| output_manifest.append(entry) | |
| src_debugs_by_target[target_label] = target_src_debugs | |
| time_profile.start( | |
| "compute_elf_build_ids", | |
| "Extra GNU build-id values from ELF binaries.", | |
| ) | |
| parser.enable_build_id_resolution() | |
| parser.parse_manifest_json(output_manifest) | |
| if _DEBUG_SYMBOL_EXPORT: | |
| print("DEBUG SYMBOLS:\n%s" % output_manifest) | |
| debug_symbols_by_target = BazelDebugSymbolMap( | |
| ( | |
| target_label, | |
| [ | |
| entries_by_src_debug[src_debug] | |
| for src_debug in target_src_debugs | |
| if entries_by_src_debug[src_debug].get("elf_build_id") | |
| ], | |
| ) | |
| for target_label, target_src_debugs in src_debugs_by_target.items() | |
| ) | |
| time_profile.stop() | |
| return parser.entries, debug_symbols_by_target | |
| def copy_debug_symbols_to_build_dir( | |
| build_dir: Path, debug_symbols_manifest: list[DebugSymbolEntryType] | |
| ) -> None: | |
| """Copy debug symbols from the Bazel execroot to {NINJA_BUILD_DIR}/.build-id | |
| Useful when performing local debugging and symbolization. This does not affect | |
| infra builds which use artifactory instead to upload the symbols to cloud | |
| storage. | |
| Args: | |
| build_dir: Path to Ninja build directory. | |
| debug_symbols_manifest: A list of DebugSymbolEntryType values, similar | |
| to what is returned by merge_debug_symbol_manifests(). | |
| """ | |
| exporter = DebugSymbolExporter( | |
| build_dir, | |
| log=lambda m: ( | |
| print(f"DEBUG: {m}", file=sys.stderr) | |
| if _DEBUG_SYMBOL_EXPORT | |
| else None | |
| ), | |
| ) | |
| exporter.parse_debug_symbols(debug_symbols_manifest) | |
| debug_copies = exporter.get_debug_symbols_to_build_id_copies( | |
| os.path.join(build_dir, ".build-id") | |
| ) | |
| def copy_build_id_file(src_path: str, dst_path: str) -> None: | |
| os.makedirs(os.path.dirname(dst_path), exist_ok=True) | |
| shutil.copyfile(src_path, dst_path) | |
| thread_pool_helpers.starmap_threaded( | |
| copy_build_id_file, | |
| debug_copies, | |
| ) |