blob: d0630d35efc945dd55ca610040d8f4b4319ebb47 [file]
# Copyright 2024 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Misc utility functions related to the Bazel workspace."""
import dataclasses
import json
import os
import shlex
import stat
import typing as T
from pathlib import Path
from textwrap import dedent
from build.bazel.scripts import bazel_invocation as bazel_inv
from build.bazel.scripts import build_utils
from build.bazel.scripts.bazel_build_flags import DefaultBuildFlagsMap
# LINT.IfChange(gn_targets_dir_symlink)
# Location of the @gn_targets redirection symlink, relative
# to the Bazel workspace.
GN_TARGETS_DIR_SYMLINK = "fuchsia_build_generated/gn_targets_dir"
# LINT.ThenChange(//build/bazel/toplevel.MODULE.bazel:gn_targets_dir)
# Separation character used by Bazel for extension-generated repo names.
#
# See https://bazel.build/external/extension#repository_names_and_visibility
_BAZEL_REPO_NAME_SEPARATOR = "+"
def workspace_should_exclude_file(path: str) -> bool:
"""Return true if a file path must be excluded from the symlink list.
Args:
path: File path, relative to the top-level Fuchsia directory.
Returns:
True if this file path should not be symlinked into the Bazel workspace.
"""
# Never symlink to the 'out' directory.
if path == "out":
return True
# Don't symlink the Jiri files, this can confuse Jiri during an 'jiri update'
# Don't symlink the .fx directory (TODO(digit): I don't remember why?)
# Don't symlink the .git or .jj directories as well.
if path.startswith((".jiri", ".fx", ".git", ".jj")):
return True
# Don't symlink the convenience symlinks from the Fuchsia source tree
if path in ("bazel-bin", "bazel-out", "bazel-repos", "bazel-workspace"):
return True
return False
def make_removable(path: str) -> None:
"""Ensure the file at |path| is removable."""
islink = os.path.islink(path)
# Skip if the input path is a symlink, and chmod with
# `follow_symlinks=False` is not supported. Linux is the most notable
# platform that meets this requirement, and adding S_IWUSR is not necessary
# for removing symlinks.
if islink and (os.chmod not in os.supports_follow_symlinks):
return
info = os.stat(path, follow_symlinks=False)
if info.st_mode & stat.S_IWUSR == 0:
try:
if islink:
os.chmod(
path, info.st_mode | stat.S_IWUSR, follow_symlinks=False
)
else:
os.chmod(path, info.st_mode | stat.S_IWUSR)
except Exception as e:
raise RuntimeError(
f"Failed to chmod +w to {path}, islink: {islink}, info: {info}, error: {e}"
)
def remove_dir(path: str) -> None:
"""Properly remove a directory."""
# shutil.rmtree() does not work well when there are readonly symlinks to
# directories. This results in weird NotADirectory error when trying to
# call os.scandir() or os.rmdir() on them (which happens internally).
#
# Re-implement it correctly here. This is not as secure as it could
# (see "shutil.rmtree symlink attack"), but is sufficient for the Fuchsia
# build.
all_files = []
all_dirs = []
for root, subdirs, files in os.walk(path):
# subdirs may contain actual symlinks which should be treated as
# files here.
real_subdirs = []
for subdir in subdirs:
if os.path.islink(os.path.join(root, subdir)):
files.append(subdir)
else:
real_subdirs.append(subdir)
for file in files:
file_path = os.path.join(root, file)
all_files.append(file_path)
make_removable(file_path)
for subdir in real_subdirs:
dir_path = os.path.join(root, subdir)
all_dirs.append(dir_path)
make_removable(dir_path)
for file in reversed(all_files):
os.remove(file)
for dir in reversed(all_dirs):
os.rmdir(dir)
os.rmdir(path)
def create_clean_dir(path: str) -> None:
"""Create a clean directory."""
if os.path.exists(path):
remove_dir(path)
os.makedirs(path)
# Type describing a callable that takes a file path as input and
# returns a content hash string for it as output.
FileHasherType: T.TypeAlias = T.Callable[[str | Path], str]
class GeneratedWorkspaceFiles(object):
"""Models the content of a generated Bazel workspace.
Usage is:
1. Create instance
2. Optionally call set_file_hasher() if recording the content hash
of input files is useful (see add_input_file()).
3. Call the record_xxx() methods as many times as necessary to
describe new workspace entries. This does not write anything
to disk.
4. Call to_json() to return a dictionary describing all added entries
so far. This can be used to compare it to the result of a previous
workspace generation.
5. Call write() to populate the workspace with the recorded entries.
"""
def __init__(self) -> None:
self._files: dict[str, T.Any] = {}
self._file_hasher: FileHasherType | None = None
self._input_files: set[Path] = set()
def set_file_hasher(self, file_hasher: FileHasherType) -> None:
self._file_hasher = file_hasher
def _check_new_path(self, path: str) -> None:
assert path not in self._files, (
"File entry already in generated list: " + path
)
@property
def input_files(self) -> set[Path]:
"""The set of input file Paths that were read through read_text_file()."""
return self._input_files
def read_text_file(self, path: Path) -> str:
"""Read an input file and return its content as text.
This also ensures that the file is tracked as an input file
to later be returned through the self.input_files property.
Args:
path: Input file path.
Returns:
content of input path, as a string.
"""
path = path.resolve()
self._input_files.add(path)
self.record_input_file_hash(path)
return path.read_text()
def record_symlink(self, dst_path: str, target_path: str | Path) -> None:
"""Record a new symlink entry.
Note that the entry always generates a relative symlink target
when writing the entry to the workspace in write(), even if
target_path is absolute.
Args:
dst_path: symlink path, relative to workspace root.
target_path: symlink target path.
"""
self._check_new_path(dst_path)
self._files[dst_path] = {
"type": "symlink",
"target": str(target_path),
}
def record_raw_symlink(
self, dst_path: str, target_path: str | Path
) -> None:
"""Record a new symlink entry.
Note that the entry always generates a relative symlink target
when writing the entry to the workspace in write(), even if
target_path is absolute.
Args:
dst_path: symlink path, relative to workspace root.
target_path: symlink target path.
"""
self._check_new_path(dst_path)
self._files[dst_path] = {
"type": "raw_symlink",
"target": str(target_path),
}
def record_file_content(
self, dst_path: str, content: str, executable: bool = False
) -> None:
"""Record a new data file entry.
Args:
dst_path: file path, relative to workspace root.
content: file content as a string.
executable: optional flag, set to True to indicate this is an
executable file (i.e. a script).
"""
self._check_new_path(dst_path)
entry: dict[str, T.Any] = {
"type": "file",
"content": content,
}
if executable:
entry["executable"] = True
self._files[dst_path] = entry
def record_input_file_hash(self, input_path: str | Path) -> None:
"""Record the content hash of an input file.
If set_file_hash_callback() was called, compute the content hash of
a given input file path and record it. Note that nothing will be written
to the workspace. This is only useful when comparing the output of
to_json() between different generations to detect when the input file
has changed.
"""
input_file = str(input_path)
self._check_new_path(input_file)
if self._file_hasher:
self._files[input_file] = {
"type": "input_file",
"hash": self._file_hasher(input_path),
}
def to_json(self) -> str:
"""Convert recorded entries to JSON string."""
return json.dumps(self._files, indent=2, sort_keys=True)
def write(self, out_dir: str | Path) -> None:
"""Write all recorded entries to a workspace directory."""
for path, entry in self._files.items():
type = entry["type"]
if type == "symlink":
target_path = entry["target"]
link_path = os.path.join(out_dir, path)
build_utils.force_symlink(link_path, target_path)
elif type == "raw_symlink":
target_path = entry["target"]
link_path = os.path.join(out_dir, path)
build_utils.force_raw_symlink(link_path, target_path)
elif type == "file":
file_path = os.path.join(out_dir, path)
os.makedirs(os.path.dirname(file_path), exist_ok=True)
with open(file_path, "w") as f:
f.write(entry["content"])
if entry.get("executable", False):
os.chmod(file_path, 0o755)
elif type == "input_file":
# Nothing to do here.
pass
else:
assert False, "Unknown entry type: " % entry["type"]
def update_if_needed(self, out_dir: Path, manifest_path: Path) -> bool:
"""Write all recorded entries if they differ from the content of a given manifest.
If the manifest exists and its content matches the recorded entries,
then this method does not do anything. Otherwise, it will overwrite
the manifests with new values, clean the output directory, and re-populate
it entirely with new content.
Args:
out_dir: Output directory to update if needed.
manifest_path: Path to manifest file to use for comparisons.
Returns:
True if the output directory and manifest were updated, False otherwise.
"""
current_manifest = self.to_json()
if out_dir.is_dir() and manifest_path.exists():
if manifest_path.read_text() == current_manifest:
# Nothing to change here.
return False
manifest_path.parent.mkdir(parents=True, exist_ok=True)
manifest_path.write_text(current_manifest)
create_clean_dir(str(out_dir))
self.write(out_dir)
return True
def record_fuchsia_workspace(
generated: GeneratedWorkspaceFiles,
top_dir: Path,
fuchsia_dir: Path,
gn_output_dir: Path,
git_bin_path: Path,
log: T.Callable[[str], None] | None = None,
) -> None:
"""Record generated entries for the Fuchsia workspace and helper files.
Note that this hards-code a few paths, i.e.:
- The generated Bazel wrapper script is written to ${top_dir}/bazel
- Bazel output base is at ${top_dir}/output_base
- Bazel output user root is at ${top_dir}/output_user_root
- The workspace goes into ${top_dir}/workspace/
The log directory is now dynamically set per fx-build invocation and
per bazel invocation in wrapper.bazel.sh.
Args:
generated: A GeneratedWorkspaceFiles instance modified by this function.
top_dir: Path to the top-level
fuchsia_dir: Path to the Fuchsia source checkout.
gn_output_dir: Path to the GN/Ninja output directory.
git_bin_path: Path to the host git binary to use during the build.
log: Optional logging callback. If not None, must take a single
string as argument.
"""
host_os = build_utils.get_host_platform()
host_cpu = build_utils.get_host_arch()
host_tag = build_utils.get_host_tag()
templates_dir = fuchsia_dir / "build" / "bazel" / "templates"
ninja_binary = (
fuchsia_dir / "prebuilt" / "third_party" / "ninja" / host_tag / "ninja"
)
bazel_bin = (
fuchsia_dir / "prebuilt" / "third_party" / "bazel" / host_tag / "bazel"
)
python_prebuilt_dir = (
fuchsia_dir / "prebuilt" / "third_party" / "python3" / host_tag
)
output_base_dir = top_dir / build_utils.BazelPaths.OUTPUT_BASE_FROM_TOP_DIR
output_user_root = (
top_dir / build_utils.BazelPaths.OUTPUT_USER_ROOT_FROM_TOP_DIR
)
workspace_dir = top_dir / build_utils.BazelPaths.WORKSPACE_FROM_TOP_DIR
bazel_launcher = (
top_dir / build_utils.BazelPaths.LAUNCHER_FROM_TOP_DIR
).resolve()
if log:
log(
f"""- Using directories and files:
Fuchsia: {fuchsia_dir}
GN build: {gn_output_dir}
Ninja binary: {ninja_binary}
Bazel source: {bazel_bin}
Top dir: {top_dir}
Logs directory: <per-invocation: out/_build_logs/OUTDIR_BASENAME/build.$date.*/bazel_logs/>
Bazel workspace: {workspace_dir}
Bazel output_base: {output_base_dir}
Bazel output user root: {output_user_root}
Bazel launcher: {bazel_launcher}
Git binary path: {git_bin_path}"""
)
def expand_template_file(
generated: GeneratedWorkspaceFiles, filename: str, **kwargs: T.Any
) -> str:
"""Expand a template file and add it to the set of tracked input files."""
template_file = templates_dir / filename
return generated.read_text_file(template_file).format(**kwargs)
def record_expanded_template(
generated: GeneratedWorkspaceFiles,
dst_path: str,
template_name: str,
**kwargs: T.Any,
) -> None:
"""Expand a template file and record its content.
Args:
generated: A GeneratedWorkspaceFiles instance.
dst_path: Destination path for the recorded expanded content.
template_name: Name of the template file to use.
executable: Optional flag, set to True to indicate an executable file.
**kwargs: Template expansion arguments.
"""
executable = kwargs.get("executable", False)
kwargs.pop("executable", None)
content = expand_template_file(generated, template_name, **kwargs)
generated.record_file_content(dst_path, content, executable=executable)
generated.record_symlink(
"workspace/MODULE.bazel",
fuchsia_dir / "build" / "bazel" / "toplevel.MODULE.bazel",
)
generated.record_symlink(
"workspace/BUILD.bazel",
fuchsia_dir / "build" / "bazel" / "toplevel.BUILD.bazel",
)
generated.record_symlink(
"workspace/REPO.bazel",
fuchsia_dir / "build" / "bazel" / "toplevel.REPO.bazel",
)
# Generate top-level symlinks
for name in os.listdir(fuchsia_dir):
if not workspace_should_exclude_file(name):
generated.record_symlink(f"workspace/{name}", fuchsia_dir / name)
# Generate a platform mapping file to ensure that using --platforms=<value>
# also sets --cpu properly, as required by the Bazel SDK rules. See comments
# in template file for more details.
_BAZEL_CPU_MAP = {"x64": "k8", "arm64": "aarch64"}
host_os = build_utils.get_host_platform()
host_cpu = build_utils.get_host_arch()
host_tag = build_utils.get_host_tag()
record_expanded_template(
generated,
"workspace/platform_mappings",
"template.platform_mappings",
bazel_host_cpu=_BAZEL_CPU_MAP.get(host_cpu, host_cpu),
host_cpu=host_cpu,
host_os=host_os,
)
# Generate the content of .bazelrc
bazelrc_content = expand_template_file(
generated,
"template.bazelrc",
)
bazelrc_generator = BazelrcFromGnConfigGenerator()
bazelrc_content += "\n" + bazelrc_generator.generate_bazelrc(gn_output_dir)
generated.record_file_content("workspace/.bazelrc", bazelrc_content)
# Copy the wrapper script to topdir/bazel, and generate the configuration
# file at topdir/bazel.sh.config that is required to run it.
# Generate wrapper script in topdir/bazel that invokes Bazel with the right --output_base.
generated.record_symlink(
"bazel", os.path.join(fuchsia_dir, "build/bazel/wrapper.bazel.sh")
)
def config_path_value(p: str | Path) -> str:
return shlex.quote(os.path.abspath(p))
record_expanded_template(
generated,
# LINT.IfChange(bazel.sh.config)
"bazel.sh.config",
# LINT.ThenChange(//build/bazel/wrapper.bazel.sh)
"template.bazel.sh.config",
bazel_bin=config_path_value(bazel_bin),
bazel_output_base=config_path_value(output_base_dir),
bazel_output_user_root=config_path_value(output_user_root),
bazel_workspace_dir=config_path_value(workspace_dir),
ninja_build_dir=config_path_value(gn_output_dir),
prebuilt_ninja=config_path_value(ninja_binary),
prebuilt_python_dir=config_path_value(python_prebuilt_dir),
bazel_logs_dirname=shlex.quote(bazel_inv.BAZEL_LOGS_DIRNAME),
)
generated.record_symlink(
os.path.join(
"workspace",
"fuchsia_build_generated",
"assembly_developer_overrides.json",
),
os.path.join(gn_output_dir, "gen", "assembly_developer_overrides.json"),
)
generated.record_symlink(
os.path.join(
"workspace",
"fuchsia_build_generated",
"icu_build_config.json",
),
os.path.join(gn_output_dir, "icu_build_config.json"),
)
# LINT.IfChange
generated.record_symlink(
"workspace/fuchsia_build_generated/args.json",
gn_output_dir / "args.json",
)
# LINT.ThenChange(//build/bazel_sdk/bazel_rules_fuchsia/common/fuchsia_platform_build.bzl)
# Create a symlink to the git host executable to make it accessible
# when running a Bazel action on bots where it is not installed in
# a standard location.
generated.record_symlink(
"workspace/fuchsia_build_generated/git",
git_bin_path,
)
# LINT.IfChange
# .jiri_root/ is not exposed to the workspace, but //build/info/BUILD.bazel
# needs to access .jiri_root/update_history/latest so create a symlink just
# for this file.
generated.record_symlink(
"workspace/fuchsia_build_generated/jiri_snapshot.xml",
fuchsia_dir / ".jiri_root" / "update_history" / "latest",
)
# LINT.ThenChange(//build/info/info.gni)
generated.record_symlink(
"workspace/fuchsia_build_generated/content_hashes",
gn_output_dir / "regenerator_outputs" / "bazel_content_hashes",
)
# Used by @fuchsia_platform_sysroot
generated.record_symlink(
"workspace/fuchsia_build_generated/fuchsia_platform_sysroot",
gn_output_dir / "regenerator_outputs" / "fuchsia_platform_sysroot",
)
# Used when merging the IDK sub-build directories. For other use cases,
# prefer a symlink with a narrower scope.
generated.record_symlink(
# LINT.IfChange
"workspace/fuchsia_build_generated/ninja_root_build_dir",
# LINT.ThenChange(//build/bazel/bazel_sdk/BUILD.bazel)
gn_output_dir,
)
generated.record_symlink(
# LINT.IfChange
"workspace/fuchsia_build_generated/fuchsia_in_tree_idk.hash",
# LINT.ThenChange(//build/bazel/bazel_sdk/BUILD.bazel, //build/bazel/toplevel.MODULE.bazel)
# LINT.IfChange
gn_output_dir / "sdk/prebuild/in_tree_collection.json",
# LINT.ThenChange(//build/regenerator.py)
)
generated.record_symlink(
# LINT.IfChange
"workspace/fuchsia_build_generated/fuchsia_internal_only_idk.hash",
# LINT.ThenChange(//build/bazel/toplevel.WORKSPACE.bzlmod)
# LINT.IfChange
gn_output_dir / "obj/build/bazel/fuchsia_internal_only_idk.hash",
# LINT.ThenChange(//build/bazel/BUILD.gn)
)
# The following symlinks are used only by bazel_gn_target_action.py when processing
# the list of Bazel source inputs, the actual repository setup in
# MODULE.bazel reuses the two symlinks above instead.
generated.record_symlink(
# LINT.IfChange
"workspace/fuchsia_build_generated/fuchsia_sdk.hash",
# LINT.ThenChange(//build/bazel/scripts/bazel_gn_target_action.py)
# LINT.IfChange
gn_output_dir / "sdk/prebuild/in_tree_collection.json",
# LINT.ThenChange(//build/regenerator.py)
)
generated.record_symlink(
# LINT.IfChange
"workspace/fuchsia_build_generated/internal_sdk.hash",
# LINT.ThenChange(//build/bazel/scripts/bazel_gn_target_action.py)
# LINT.IfChange
gn_output_dir / "obj/build/bazel/fuchsia_internal_only_idk.hash",
# LINT.ThenChange(//build/bazel/BUILD.gn)
)
# Create a link to an empty repository. This is updated by bazel_gn_target_action.py
# before each Bazel invocation to point to the @gn_targets content specific
# to its parent bazel_action() target.
generated.record_symlink(
f"workspace/{GN_TARGETS_DIR_SYMLINK}",
fuchsia_dir
/ "build"
/ "bazel"
/ "local_repositories"
/ "empty_gn_targets",
)
generated.record_symlink(
# LINT.IfChange
"workspace/fuchsia_build_generated/fuchsia_build_info",
# LINT.ThenChange(//build/bazel/toplevel.MODULE.bazel)
# LINT.IfChange
gn_output_dir / "regenerator_outputs/fuchsia_build_info",
# LINT.ThenChange(//build/regenerator.py)
)
generated.record_symlink(
# LINT.IfChange
"workspace/fuchsia_build_generated/fuchsia_in_tree_idk",
# LINT.ThenChange(//build/bazel/toplevel.MODULE.bazel)
# LINT.IfChange
gn_output_dir / "regenerator_outputs/fuchsia_in_tree_idk",
# LINT.ThenChange(//build/regenerator.py)
)
_record_abseil_cpp_repository(generated, fuchsia_dir)
def _record_abseil_cpp_repository(
generated: GeneratedWorkspaceFiles,
fuchsia_dir: Path,
) -> None:
"""Record the @abseil-cpp repository in workspace/fuchsia_build_generated/abseil-cpp.
Upstream abseil-cpp's absl/time/internal/cctz/BUILD.bazel omits the Fuchsia
dependencies required by time_zone_lookup.cc when __Fuchsia__ is defined.
Overlay that single BUILD.bazel file while symlinking all other files and
directories directly to //third_party/abseil-cpp/src.
"""
abseil_src = fuchsia_dir / "third_party" / "abseil-cpp" / "src"
cctz_build_rel = "absl/time/internal/cctz/BUILD.bazel"
cctz_build_src = abseil_src / cctz_build_rel
if not cctz_build_src.exists():
return
# LINT.IfChange
dst_root = "workspace/fuchsia_build_generated/abseil-cpp"
# LINT.ThenChange(//build/bazel/toplevel.MODULE.bazel)
expand_dirs = {
"",
"absl",
"absl/time",
"absl/time/internal",
"absl/time/internal/cctz",
}
for rel_dir in sorted(expand_dirs):
src_dir = abseil_src / rel_dir if rel_dir else abseil_src
for name in sorted(os.listdir(src_dir)):
rel_path = f"{rel_dir}/{name}" if rel_dir else name
if rel_path in expand_dirs:
continue
dst_path = f"{dst_root}/{rel_path}"
if rel_path == cctz_build_rel:
content = generated.read_text_file(cctz_build_src)
old_select = """\
deps = [
":civil_time",
"//absl/base:config",
] + select(
{
"//conditions:default": [],
},
),"""
new_select = """\
deps = [
":civil_time",
"//absl/base:config",
] + select(
{
"@platforms//os:fuchsia": [
"@@//sdk/fidl/fuchsia.intl:fuchsia.intl_hlcpp",
"@@//sdk/lib/async-loop:async-loop-cpp",
"@@//sdk/lib/fdio",
],
"//conditions:default": [],
},
),"""
assert old_select in content, (
f"Expected select() block not found in {cctz_build_src}. "
"If a new abseil-cpp was rolled into the Fuchsia tree, "
f"please update `_record_abseil_cpp_repository` in {__file__}"
)
generated.record_file_content(
dst_path, content.replace(old_select, new_select, 1)
)
else:
generated.record_symlink(dst_path, src_dir / name)
def generate_fuchsia_workspace(
fuchsia_dir: Path,
build_dir: Path,
log: T.Callable[[str], None] | None = None,
) -> set[Path]:
"""Generate the Fuchsia Bazel workspace and associated files.
Args:
fuchsia_dir: Path to the Fuchsia source checkout.
build_dir: Path to the GN/Ninja output directory.
log: Optional logging callback. If not None, must take a single
string as argument.
Returns:
A set of input file paths that were read by this function.
"""
# Find path of host git tool
git_bin_path = build_utils.find_host_binary_path("git")
assert git_bin_path, f"Could not find 'git' in current PATH!"
# # Extract target_cpu from args.json. This file is GN-generated
# # and doesn't need to be added to input_files.
# args_json_path = build_dir / "args.json"
# assert args_json_path.exists(), f"Missing GN output file: {args_json_path}"
# with args_json_path.open("rb") as f:
# args_json = json.load(f)
#
# target_cpu = args_json.get("target_cpu")
# assert target_cpu, f"Missing target_cpu key in {args_json_path}"
# Find the location of the Bazel top-dir relative to the Ninja
# build directory.
bazel_top_dir, input_files = build_utils.get_bazel_relative_topdir(
fuchsia_dir
)
# Generate the bazel launcher and Bazel workspace files.
generated = GeneratedWorkspaceFiles()
top_dir = build_dir / bazel_top_dir
record_fuchsia_workspace(
generated,
top_dir=top_dir,
fuchsia_dir=fuchsia_dir,
gn_output_dir=build_dir,
git_bin_path=git_bin_path,
log=log,
)
# Only recreate the workspace directory and clean `output_base/external`
# when the generated workspace manifest actually changes. Unconditionally
# unlinking `workspace` deletes the running Bazel server's working
# directory (forcing a daemon restart on every `fx gen`), and wiping
# `output_base/external` forces Bazel to re-fetch all external
# repositories during `tests.json` generation.
generated_json = generated.to_json()
generated_info_file = top_dir / "generated-info.json"
workspace_dir = top_dir / "workspace"
if (
not workspace_dir.is_dir()
or not generated_info_file.exists()
or generated_info_file.read_text() != generated_json
):
# Remove the old workspace's content, which is just a set
# of symlinks and some auto-generated files.
create_clean_dir(str(workspace_dir))
# Do not clean the output_base because it is now massive,
# and doing this will very slow and will force a lot of
# unnecessary rebuilds after that,
#
# However, do remove $OUTPUT_BASE/external/ which holds
# the content of external repositories.
#
# This is a guard against repository rules that do not
# list their input dependencies properly, and would fail
# to re-run if one of them is updated. Sadly, this is
# pretty common with Bazel.
create_clean_dir(os.path.join(top_dir / "output_base" / "external"))
generated.write(top_dir)
generated_info_file.write_text(generated_json)
return input_files | generated.input_files
class GnBuildArgs(object):
"""A class to handle `gn_build_variables_for_bazel.json` input files.
These files are used to export the values of GN args to an `args.bzl` file
and optionally `vendor_<name>_args.bzl` files in the @fuchsia_build_info
Bazel repository.
See the comments for //build/bazel:gn_build_variables_for_bazel for the
format of the `gn_build_variables_for_bazel.json` files.
Vendor-specific files should always be placed at
$OUT_DIR/vendor_<name>_gn_build_variables_for_bazel.json
and the target that generates them must always be in the dependency graph
when building anything from the vendor repo.
"""
@staticmethod
def find_all_gn_build_variables_for_bazel(
fuchsia_dir: Path, build_dir: Path
) -> list[str]:
"""Find `all gn_build_variables_for_bazel.json` files in the Fuchsia checkout.
Args:
fuchsia_dir: Path to Fuchsia source dir.
build_dir: Path to Fuchsia build directory.
Returns:
A list of `gn_build_variables_for_bazel.json` file paths relative to
the build directory root.
Raises:
ValueError if a required input file is missing.
"""
args_files: list[str] = []
base_file_name = "gn_build_variables_for_bazel.json"
main_file_path = build_dir / base_file_name
if not (main_file_path).exists():
raise ValueError(
f"Missing required build arguments file: {main_file_path}"
)
args_files.append(base_file_name)
# The //vendor/ tree is optional and does not exist in open-source checkouts.
vendor_dir = fuchsia_dir / "vendor"
if vendor_dir.is_dir():
for vendor_name in os.listdir(vendor_dir):
vendor_file = f"vendor_{vendor_name}_{base_file_name}"
if (build_dir / vendor_file).exists():
args_files.append(vendor_file)
return args_files
@staticmethod
def generate_args_bzl(
gn_args_to_export: list[dict[str, T.Any]], args_json_path: Path
) -> str:
"""Generate an `args.bzl` file defining values extracted from GN's args.
Args:
gn_args_to_export: A list of dictionaries describing each GN arg.
See comments for //build/bazel:gn_build_variables_for_bazel for
the format.
args_json_path: Path of source file for build_args (never accessed).
Returns:
The content of the generated `args.bzl` file as a string.
Raises:
ValueError if the input content is malformed.
"""
def fail(msg: str) -> None:
raise ValueError(msg)
args_contents = """# AUTO-GENERATED BY FUCHSIA BUILD - DO NOT EDIT
# Variables listed from {source_path}
\"\"\"A subset of GN args that are needed in the Bazel build.\"\"\"
""".format(
source_path=args_json_path
)
for gn_arg in gn_args_to_export:
args_contents += "\n# From {}\n".format(gn_arg["location"])
varname = gn_arg["name"]
vartype = gn_arg["type"]
value = gn_arg["value"]
if vartype in ["bool", "array_of_strings"]:
args_contents += "{} = {}".format(varname, value)
elif vartype in ["string", "string_or_false", "path"]:
if vartype == "string_or_false" and not value:
value = ""
elif vartype == "path":
if value.startswith("//"):
value = value[2:]
elif value.startswith("/"):
# Pass through the absolute path.
value = value
else:
fail(
"Path '{}' does not begin with '//' or '/'".format(
value,
)
)
args_contents += '{} = "{}"'.format(varname, value)
else:
fail(
"Unknown type name '{}'".format(
vartype,
)
)
args_contents += "\n"
return args_contents
@staticmethod
def generate_platforms_flags_bzl(
build_dir: Path,
) -> str:
"""Generate a `platforms_flags.bzl` file defining values extracted from GN's args.
Args:
build_dir: Path of source file for build_args (never accessed).
Returns:
The content of the generated `platforms_flags.bzl` file as a string.
Raises:
ValueError if the input content is malformed.
"""
# When variants are enabled for Fuchsia (e.g. with `select_variant = [ "asan" ]`),
# Fuchsia artifacts are built in a variant-specific toolchain which matches the
# default toolchain (e.g. //build/toolchain/fuchsia:arm64-asan instead of
# //build/toolchain/fuchsia:arm64). Similarly, for host variants, a variant-specific
# host toolchain will be used (e.g. `//build/toolchain:host_x64-asan` instead of
# `//build/toolchain:host_x64`). These are referenced by the special values
# fuchsia_default_variant_toolchain and host_default_variant_toolchain.
#
# Ensure all Bazel configuration flags that are defined in
# //build/bazel/config/bazel_args.gni, which is evaluated in the
# {fuchsia,host}_default_variant_toolchain contexts, are written to
# @fuchsia_build_info//:platform_flags.bzl.
#
# This file should expose a single dictionary mapping one of "host", "fuchsia" or
# "fuchsia_platform" to a lista flags (e.g. '[ "--//some/flag:label=value" ]').
#
# This in turn should be used by in-tree platform definitions
# under //build/bazel/platforms/BUILD.bazel to ensure these
# flags are set properly for each platform, even when transitioning
# from one to the other.
config_values: dict[str, list[str]] = {}
has_lto_values: dict[str, bool] = {}
for config_name in ("host", "fuchsia"):
# LINT.IfChange(bazel_args_json_path)
config_file = build_dir / "bazel_args" / f"{config_name}.json"
# LINT.ThenChange(//build/bazel/config/BUILD.gn:bazel_args_json_path)
assert (
config_file.exists()
), f"Missing 'gn gen' generated file: {config_file}"
with config_file.open("rb") as f:
info = json.load(f)
# LINT.IfChange(bazel_args_json_schema)
flags = info.get("build")
if not isinstance(flags, list):
raise ValueError(f"{config_file}: Malformed input file")
config_values[config_name] = flags
has_lto_values[config_name] = bool(info.get("has_lto", False))
# LINT.ThenChange(//build/bazel/config/bazel_args.gni:bazel_args_json_schema)
# The following map must match the platforms defined in
# //build/bazel/platforms/BUILD.bazel that get their
# 'flags' value from constant map generated by this function.
# LINT.IfChange(bazel_platforms_list)
BAZEL_PLATFORMS_TO_CONFIG = {
"linux_x64": "host",
"linux_arm64": "host",
"linux_starnix_x64": "host",
"linux_starnix_arm64": "host",
"linux_starnix_riscv64": "host",
"linux_starnix_arm": "host",
"fuchsia_sdk_x64": "fuchsia",
"fuchsia_sdk_arm64": "fuchsia",
"fuchsia_sdk_riscv64": "fuchsia",
"fuchsia_platform_x64": "fuchsia",
"fuchsia_platform_arm64": "fuchsia",
"fuchsia_platform_riscv64": "fuchsia",
}
# LINT.ThenChange(//build/bazel/platforms/BUILD.bazel)
fuchsia_has_lto = has_lto_values.get("fuchsia", False)
host_has_lto = has_lto_values.get("host", False)
content = f"""# AUTO-GENERATED BY FUCHSIA BUILD - DO NOT EDIT
fuchsia_has_lto = {fuchsia_has_lto}
host_has_lto = {host_has_lto}
platform_flags_map = {{
"""
for platform, config in BAZEL_PLATFORMS_TO_CONFIG.items():
flags = list(config_values.get(config, []))
# Force the value of --cpu=<name>. Without this, the build will fail
# when trying to build fuchsia_package() targets using the SDK rules
# due to the way the internal fuchsia_transition() transition is
# implemented.
cpu = platform.split("_")[-1]
bazel_cpu = {
"x64": "x86_64",
"arm64": "aarch64",
}.get(cpu, cpu)
flags += [f"--cpu={bazel_cpu}"]
content += ' "{}": [\n'.format(platform)
for flag in flags:
content += ' "{}",\n'.format(flag)
content += " ],\n"
content += "}\n"
return content
@staticmethod
def record_fuchsia_build_info_dir(
fuchsia_dir: Path,
build_dir: Path,
generated: GeneratedWorkspaceFiles,
) -> None:
"""Record the content of @fuchsia_build_info in a GeneratedWorkspaceFiles instance.
Args:
fuchsia_dir: Path to Fuchsia source directory.
build_dir: Path to Fuchsia build directory.
generated: A GeneratedWorkspaceFiles instance. Its record_file_content()
method will be called to populate the repository.
"""
generated.record_file_content(
"MODULE.bazel",
"""\
module(name = "fuchsia_build_info", version = "1")
bazel_dep(name = "platforms", version = "1.0.0")
bazel_dep(name = "fuchsia_rules_common", version = "0.0")
""",
)
generated.record_file_content("BUILD.bazel", "")
# Generate BUILD.bazel which contains the definitions of toolchains
# for default build_flags().
default_flags_map = DefaultBuildFlagsMap.new_from_gn_config(
build_dir, fuchsia_dir
)
generated.record_file_content(
"default_build_flags/BUILD.bazel",
default_flags_map.generate_bazel_toolchain_definitions(),
)
# Generate the args.bzl and vendor_{name}_args.bzl files
args_files_relative_paths = (
GnBuildArgs.find_all_gn_build_variables_for_bazel(
fuchsia_dir, build_dir
)
)
for relative_path in sorted(args_files_relative_paths):
file_path = build_dir / relative_path
with (file_path).open("rb") as f:
gn_args_json = json.load(f)
args_bzl_content = GnBuildArgs.generate_args_bzl(
gn_args_json, file_path
)
if relative_path.startswith("vendor_"):
vendor_name = relative_path.split("_")[1]
args_bzl_filename = f"vendor_{vendor_name}_args.bzl"
else:
args_bzl_filename = "args.bzl"
generated.record_file_content(args_bzl_filename, args_bzl_content)
# Generate @fuchsia_build_info//:platform_flags.bzl
platforms_flags_content = GnBuildArgs.generate_platforms_flags_bzl(
build_dir
)
generated.record_file_content(
"platforms_flags.bzl", platforms_flags_content
)
@staticmethod
def generate_fuchsia_build_info(
fuchsia_dir: Path, build_dir: Path, repository_dir: Path
) -> None:
generated = GeneratedWorkspaceFiles()
GnBuildArgs.record_fuchsia_build_info_dir(
fuchsia_dir, build_dir, generated
)
generated.update_if_needed(
repository_dir, Path(f"{repository_dir}.generated-info.json")
)
@dataclasses.dataclass
class GnTargetsDirectoryManifestEntry(object):
"""A single entry in the manifest describing @gn_targets,
corresponding to a single GN bazel_input_directory() target.
All paths are relative to the Ninja build directory.
Fields:
bazel_package: Bazel package name (without leading // prefix).
bazel_name: Bazel target name.
generator_label: GN target label that generates the file.
license_spdx_file: Path to an SPDX JSON file describing the license for all
outputs of the generator target. This file is generated by the
generated_licenses_spdx() GN template.
output_files: List of Ninja build artifacts.
Only non-empty for bazel_input_file() GN targets.
output_directory: Path to the directory containing Ninja output files.
Only non-empty for bazel_input_directory() GN targets.
"""
# LINT.IfChange(GnTargetsDirectoryManifestEntry)
bazel_package: str
bazel_name: str
generator_label: str
license_spdx_file: str
output_files: list[str] = dataclasses.field(default_factory=list)
output_directory: str = ""
# LINT.ThenChange(//build/bazel/bazel_inputs.gni:GnTargetsDirectoryManifestEntry)
@staticmethod
def from_json_value(json_data: T.Any) -> "GnTargetsDirectoryManifestEntry":
"""Create new instance from JSON value."""
assert isinstance(json_data, dict)
assert "bazel_package" in json_data
assert "bazel_name" in json_data
assert "generator_label" in json_data
assert "license_spdx_file" in json_data
assert ("output_files" in json_data) != (
"output_directory" in json_data
), "One, and only one, of output_files or output_directory must be defined"
return GnTargetsDirectoryManifestEntry(
bazel_package=json_data["bazel_package"],
bazel_name=json_data["bazel_name"],
generator_label=json_data["generator_label"],
license_spdx_file=json_data["license_spdx_file"],
output_files=json_data.get("output_files", []),
output_directory=json_data.get("output_directory", ""),
)
@dataclasses.dataclass
class GnTargetsDirectoryManifest(object):
"""The manifest describing @gn_targets, corresponding to a single GN bazel_action() target."""
entries: list[GnTargetsDirectoryManifestEntry]
@staticmethod
def from_json_value(json_data: T.Any) -> "GnTargetsDirectoryManifest":
"""Create new instance from JSON value."""
assert isinstance(json_data, list)
return GnTargetsDirectoryManifest(
entries=[
GnTargetsDirectoryManifestEntry.from_json_value(entry)
for entry in json_data
],
)
class BazelTargetGnInputsEntriesMap(dict[str, GnTargetsDirectoryManifestEntry]):
"""Maps Bazel target names to a GnTargetsDirectoryManifestEntry instance."""
class BazelPackageAndTargetToGnInputsEntriesMap(
dict[str, BazelTargetGnInputsEntriesMap]
):
"""Maps Bazel package names to a BazelTargetGnInputsEntriesMap instance."""
def merge_gn_target_manifests(
manifests: list[Path],
) -> BazelPackageAndTargetToGnInputsEntriesMap:
"""Merge several gn_targets manifests into a single set of entries.
Args:
manifests: Paths of the per-bazel_action() gn_targets manifests.
Returns:
The merged entries, keyed by Bazel package then by target name.
Raises:
ValueError if two manifests disagree about the same Bazel target.
"""
manifest_entries_package_map = BazelPackageAndTargetToGnInputsEntriesMap()
for manifest_path in manifests:
with open(manifest_path) as f:
for entry_json in json.load(f):
entry = GnTargetsDirectoryManifestEntry.from_json_value(
entry_json
)
bazel_package = entry.bazel_package
name_map = manifest_entries_package_map.setdefault(
bazel_package, BazelTargetGnInputsEntriesMap()
)
bazel_name = entry.bazel_name
found_entry = name_map.setdefault(bazel_name, entry)
if found_entry != entry:
raise ValueError(
f"Found duplicate GN target entry for //{bazel_package}:{bazel_name}: {found_entry.generator_label} vs {entry.generator_label}"
)
return manifest_entries_package_map
def record_gn_targets_dir_from_entries(
generated: GeneratedWorkspaceFiles,
build_dir: Path,
manifest_entries_package_map: BazelPackageAndTargetToGnInputsEntriesMap,
all_licenses_spdx_path: Path,
) -> None:
# Ensure build_dir is absolute. Most symlink targets must be absolute for Bazel
# to work properly.
build_dir = build_dir.resolve()
if not all_licenses_spdx_path.exists():
raise ValueError(
f"Missing licensing information file: {all_licenses_spdx_path}"
)
# This creates two sets of symlinks.
#
# The first one maps `_files/{ninja_path}` to the absolute path of the corresponding
# Ninja artifact in the build directory, e.g.:
#
# _files/obj/src/foo/foo.cc.o ----> $NINJA_BUILD_DIR/obj/src/foo/foo.cc
#
# There is one such symlink per Ninja output paths.
#
# Second, for each bazel_package value, `{bazel_package}/_files` will be a relative
# symlink that points to the top-level `_files` directory, as in:
#
# src/foo/_files ---> ../../_files
#
# This is used by the BUILD.bazel file generated in the same sub-directory, that can
# reference the artifacts using labels like "_files/{ninja_path}" without having
# to care for Bazel package boundaries, as in:
#
# ```
# # Generated as src/foo/BUILD.bazel
# filegroup(
# name = "foo",
# srcs = [ "_files/obj/src/foo/foo.cc.o" ]
# )
# ```
#
# The reason why this double indirection exists is purely for debuggability!
# It is easier to see all the Ninja artifacts exposed at once from the top-level
# _files/ directory when verifying correctness.
#
# It is perfectly possible to only place absollute symlinks under
# {bazel_package}/_files/... but doing this leads to repositories that are
# harder to inspect in practice due to the extra long paths it creates.
# The top-level directory that will contain symlinks to all Ninja output
# files, using . For example _files/obj/src/foo/foo.cc.o
files_subdir = "_files"
def record_file_symlink(file: str) -> str:
"""Record a symlink to a file in the files directory.
Args:
file: A file path, relative to the Ninja build directory.
Returns:
The symlink's path, relative to the generated output directory.
"""
# Create //_files/{ninja_path} as a symlink to the Ninja output location.
target_file = build_dir / file
if not target_file.exists():
# If the target does not exist, create an empty file with a timestamp of 0
# This ensures Bazel queries will not error, and that the next Ninja
# invocation will regenerate the file.
target_file.parent.mkdir(parents=True, exist_ok=True)
target_file.write_text("")
os.utime(target_file, ns=(0, 0))
link_file = f"{files_subdir}/{file}"
generated.record_raw_symlink(
link_file,
target_file,
)
return link_file
all_license_files: list[str] = []
# Create the ///{gn_dir}/BUILD.bazel file for each GN directory.
# Every target defined in {gn_dir}/BUILD.gn that is part of the manifest
# will have its own filegroup() entry with the corresponding target name.
for bazel_package, name_map in manifest_entries_package_map.items():
content = dedent(
"""\
# AUTO-GENERATED - DO NOT EDIT
load("@rules_license//rules:license.bzl", "license")
package(default_visibility = ["//visibility:public"])
"""
)
for bazel_name, entry in name_map.items():
# Create the license() target for this entry, pointing to the SPDX
# file that will be generated by Ninja.
license_symlink = record_file_symlink(entry.license_spdx_file)
license_target_name = f"{bazel_name}.license"
content += dedent(
"""\
# From GN target: {label}
license(
name = "{license_target_name}",
package_name = "Legacy Ninja Build Outputs",
license_text = "{license_symlink}",
)
"""
).format(
label=entry.generator_label,
license_target_name=license_target_name,
license_symlink=license_symlink,
)
all_license_files.append(license_symlink)
file_links = entry.output_files
if file_links:
for file in file_links:
record_file_symlink(file)
content += dedent(
"""\
filegroup(
name = "{name}",
applicable_licenses = [":{license_target_name}"],
srcs = """
).format(
name=bazel_name,
license_target_name=license_target_name,
)
if len(file_links) == 1:
content += '["_files/%s"],\n' % file_links[0]
else:
content += "[\n"
for file in file_links:
content += ' "_files/%s",\n' % file
content += " ],\n"
content += ")\n"
dir_link = entry.output_directory
if dir_link:
# Create //_files/{ninja_path} as a symlink to the real path.
target_dir = build_dir / dir_link
generated.record_raw_symlink(
f"{files_subdir}/{dir_link}", target_dir
)
# NOTE: allow_empty is necessary when performing queries
# before building anything. The glob() might return an empty
# list in this case since there is nothing built from Ninja yet.
content += dedent(
"""\
filegroup(
name = "{name}",
applicable_licenses = [":{license_target_name}"],
srcs = glob(["{ninja_path}/**"], exclude_directories=1, allow_empty=True),
)
alias(
name = "{name}.directory",
actual = "{ninja_path}",
)
"""
).format(
name=bazel_name,
license_target_name=license_target_name,
ninja_path=f"{files_subdir}/{dir_link}",
)
generated.record_file_content(f"{bazel_package}/BUILD.bazel", content)
# Because {bazel_package}/BUILD.bazel contains label references
#
# such as "_files/obj/src/microfuchsia/tee/ta/noop/ta-noop.far", create
# {bazel_package}/_files as a symlink to the top-level _files directory.
#
# A relative path target is required, as the final output directory path is
# not known yet. This much walk back the bazel_package path fragments.
generated.record_raw_symlink(
f"{bazel_package}/{files_subdir}",
("../" * len(bazel_package.split("/"))) + files_subdir,
)
# The symlink for the special all_licenses_spdx.json file.
# IMPORTANT: This must end in `.spdx.json` for license classification to work correctly!
# LINT.IfChange(all_licenses_spdx_path)
generated.record_symlink(
"all_licenses.spdx.json", all_licenses_spdx_path.resolve()
)
# LINT.ThenChange(//build/bazel/bazel_action_utils.py:all_licenses_spdx_path)
# LINT.IfChange(all_license_files)
# The list of all license file paths. This is later used to check that they were
# properly updated by Ninja
generated.record_file_content(
"all_license_files.txt", "\n".join(sorted(all_license_files))
)
# LINT.ThenChange(//build/bazel/scripts/bazel_action_utils.py:all_license_files)
# The content of BUILD.bazel
build_content = dedent(
"""\
# AUTO-GENERATED - DO NOT EDIT
load("@rules_license//rules:license.bzl", "license")
# This contains information about all the licenses of all
# Ninja outputs exposed in this repository.
# IMPORTANT: package_name *must* be "Legacy Ninja Build Outputs"
# as several license pipeline exception files hard-code this under //vendor/...
license(
name = "all_licenses_spdx_json",
package_name = "Legacy Ninja Build Outputs",
license_text = "all_licenses.spdx.json",
visibility = ["//visibility:public"]
)
"""
)
generated.record_file_content("BUILD.bazel", build_content)
generated.record_file_content(
"MODULE.bazel",
dedent(
"""\
# AUTO-GENERATED - DO NOT EDIT
module(name = "gn_targets", version = "1")
bazel_dep(name = "rules_license", version = "1.0.0")"""
),
)
class BazelrcFromGnConfigGenerator(object):
"""Generate a .bazelrc file fragment that contains --config=<name> definitions."""
_DEFAULT_PLATFORMS = (
"host",
"linux_x64",
"linux_arm64",
"linux_starnix_x64",
"linux_starnix_arm64",
"linux_starnix_riscv64",
"linux_starnix_arm",
"fuchsia",
"fuchsia_sdk",
"fuchsia_sdk_x64",
"fuchsia_sdk_arm64",
"fuchsia_sdk_riscv64",
"fuchsia_platform",
"fuchsia_platform_x64",
"fuchsia_platform_arm64",
"fuchsia_platform_riscv64",
)
def __init__(
self, platform_names: T.Iterable[str] = _DEFAULT_PLATFORMS
) -> None:
self._platform_names = platform_names
def generate_bazelrc(self, build_dir: Path) -> str:
"""Generate a .bazelrc file that contains platform-specific definitions.
This takes as input the GN-generated files from //build/bazel:bazel_args_json
and produces a .bazelrc file that contains definitions for the following:
--config=host: Matching the current host platform.
--config=fuchsia_sdk_x64, --config=fuchsia_sdk_arm64, --config=fuchsia_sdk_riscv64
--config=fuchsia_platform_x64, --config=fuchsia_platform_arm64, --config=fuchsia_platform_riscv64
Matching different Fuchsia platforms.
Args:
build_dir: The Ninja build directory.
Returns:
The content of the generated .bazelrc file fragment.
"""
# For each config_name value, create a .bazelrc fragment that will
# define Bazel flags to use by default with --config=${config_name}
output = (
"# Auto-generated lists of --config=<name> settings, do not edit!\n"
)
def to_string(l: T.Any) -> str:
assert isinstance(
l, list
), f"Expected list of strings, got {type(l)} instead!"
return " ".join(l)
config_names = ("host", "fuchsia")
config_values: dict[str, dict[str, str]] = {}
for config_name in config_names:
config_file = build_dir / f"bazel_args/{config_name}.json"
assert (
config_file.exists()
), f"Missing GN-generated input file: {config_file}"
with config_file.open() as f:
config = json.load(f)
common_args = to_string(config["common"])
output += f"common:{config_name}_config_args {common_args}\n"
config_values[config_name] = config
for platform in self._platform_names:
config_args_name = (
"fuchsia_config_args"
if platform.startswith("fuchsia")
else "host_config_args"
)
if platform in ("fuchsia_sdk", "fuchsia_platform", "fuchsia"):
# As a special case, make "--config=fuchsia_sdk" and
# "--config=fuchsia_platform" the same as
# "--config=fuchsia_sdk_${target_cpu}" and
# "--config=fuchsia_platform_${target_cpu}", respectively.
platform_base = "fuchsia"
values = config_values[platform_base]
current_cpu = values["current_cpu"]
platform_name = f"{platform}_{current_cpu}"
# As a very special case, allow the legacy config name "fuchsia"
# to be used as an alias for "fuchsia_sdk" until we have
# migrated people off of it. Override `platform` since there are
# no "fuchsia_$CPU" platforms.
if platform == "fuchsia":
platform_name = f"fuchsia_sdk_{current_cpu}"
else:
platform_name = platform
output += f"common:{platform} --config={config_args_name} --platforms=//build/bazel/platforms:{platform_name}\n"
return output
def check_regenerator_inputs_updates(
build_dir: Path,
inputs_file: str = "regenerator_outputs/regenerator_inputs.txt",
) -> set[str]:
"""Check whether any regenerator input has changed.
Args:
build_dir: Ninja build directory path.
inputs_file: Optional path string to the file listing all inputs,
relative to build_dir. Default value is
regenerator_outputs/regenerator_inputs.txt.
Returns:
A set of file paths, relative to build_dir, whose timestamp is
newer than that of the inputs_file file itself.
"""
inputs_path = build_dir / inputs_file
if not inputs_path.exists():
# If the file does not exist, a regeneration is required.
return {inputs_file}
inputs_timestamp = inputs_path.stat().st_mtime
changed_inputs: set[str] = set()
for dep in inputs_path.read_text().splitlines():
dep_path = build_dir / dep
if not dep_path.exists():
changed_inputs.add(dep)
continue
dep_timestamp = dep_path.stat().st_mtime
if dep_timestamp > inputs_timestamp:
changed_inputs.add(dep)
return changed_inputs
def _extract_variant_and_subpath(dest: str, prefix: str) -> tuple[str, str]:
"""Extracts (variant, subpath) from a destination path.
E.g. "dist/lib/ld.so.1" -> ("", "ld.so.1")
"dist/lib/asan/ld.so.1" -> ("asan", "asan/ld.so.1")
"""
subpath = dest.removeprefix(prefix)
variant = subpath.split("/")[0] if "/" in subpath else ""
return variant, subpath
def _generate_sysroot_dist_targets(
dist_files: dict[str, tuple[str, str]],
debug_files: dict[str, str],
bazel_arch: str,
) -> list[str]:
"""Generates target declarations for sysroot dist files.
Args:
dist_files: A dict mapping from variant name to a tuple
(stripped_file, subpath), where stripped_file is the path to the
stripped file relative to the sysroot directory, and subpath is the
path under `lib` to install the stripped file.
debug_files: A dict mapping from variant name to the path to the
unstripped file relative to the sysroot directory.
bazel_arch: The target CPU architecture (in Bazel's cpu representation, e.g. "x86_64")
Returns:
A list of target declarations of fuchsia_unstripped_binary targets and
a variant-selecting sysroot_library_dist alias.
"""
dist_targets: list[str] = []
select_entries: dict[str, str] = {}
for variant in sorted(dist_files.keys()):
if variant not in debug_files:
continue
variant_suffix = variant if variant else "novariant"
target_name = f"sysroot_library_dist.{variant_suffix}"
stripped_file, subpath = dist_files[variant]
unstripped_file = debug_files[variant]
dist_targets.append(
dedent(
f"""\
fuchsia_unstripped_binary(
name = "{target_name}",
dest = "lib/{subpath}",
stripped_file = "{stripped_file}",
unstripped_file = "{unstripped_file}",
target_compatible_with = [
"@platforms//os:fuchsia",
"@platforms//cpu:{bazel_arch}",
],
visibility = ["//visibility:public"],
)"""
)
)
# LINT.IfChange(sysroot_variants)
if variant:
select_entries[
f"@@//build/bazel/toolchains/clang:{variant}_variant"
] = f":{target_name}"
if "" in dist_files and "" in debug_files:
select_entries[
"@@//build/bazel/toolchains/clang:novariant"
] = ":sysroot_library_dist.novariant"
# LINT.ThenChange(//build/bazel/toolchains/clang/BUILD.bazel:sysroot_variants)
if select_entries:
select_dict_lines = "".join(
f' "{cond}": "{target}",\n'
for cond, target in select_entries.items()
)
dist_targets.append(
f"""\
alias(
name = "sysroot_library_dist",
actual = select({{
{select_dict_lines} }}),
visibility = ["//visibility:public"],
)"""
)
return dist_targets
def generate_fuchsia_platform_sysroot_repository(
repository_dir: Path,
repository_name: str,
sysroot_json: Path,
gn_target_cpu: str,
build_dir: Path | None = None,
) -> None:
"""Generate a Bazel repository that contains a sysroot for the Fuchsia platform artifacts.
Args:
repository_dir: Output directory for the generated Bazel repository.
repository_name: Name of the generated repository.
sysroot_json: Input sysroot JSON file, such as the one generated by
//zircon/public/sysroot_sdk:sysroot_for_fuchsia_platform.
gn_target_cpu: The GN target CPU architecture.
build_dir: Optional path to the Ninja build directory. Defaults to the parent
directory of --sysroot_json.
"""
sysroot_json = sysroot_json.resolve()
repository_dir = repository_dir.resolve()
if build_dir is None:
build_dir = sysroot_json.parent
build_dir = build_dir.resolve()
header_files: list[str] = []
lib_files: list[str] = []
# Paths to symlinked stripped files and debug files, relative to the repository root.
# These files are later used as sources for fuchsia_unstripped_binary targets.
dist_files: dict[str, tuple[str, str]] = {}
debug_files: dict[str, str] = {}
with sysroot_json.open("r") as f:
sysroot_json_data = json.load(f)
dest_prefix = "sysroot/"
for entry in sysroot_json_data:
source_path = build_dir / entry["source"]
symlink_dest = entry["dest"]
symlink_dest_path = dest_prefix + symlink_dest
build_utils.force_symlink_to_ninja_artifact(
repository_dir / symlink_dest_path, source_path
)
if symlink_dest.startswith("include/"):
header_files.append(symlink_dest_path)
elif symlink_dest.startswith("lib/"):
lib_files.append(symlink_dest_path)
elif symlink_dest.startswith("dist/lib/"):
assert symlink_dest.endswith(
"ld.so.1"
), f"Unexpected file in dist/lib/: {symlink_dest}"
variant, subpath = _extract_variant_and_subpath(
symlink_dest, "dist/lib/"
)
dist_files[variant] = (symlink_dest_path, subpath)
elif symlink_dest.startswith("debug/"):
assert symlink_dest.endswith(
"libc.so"
), f"Unexpected file in debug/: {symlink_dest}"
variant, _ = _extract_variant_and_subpath(symlink_dest, "debug/")
debug_files[variant] = symlink_dest_path
def write_file_if_changed(path: Path, content: str) -> None:
if not path.exists() or path.read_text() != content:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
# As Bazel doesn't support labels pointing to directories, this empty file can
# be referenced as @<repo_name>//:sysroot/empty instead.
repository_dir.mkdir(parents=True, exist_ok=True)
write_file_if_changed(repository_dir / "sysroot/empty", "")
dist_targets = _generate_sysroot_dist_targets(
dist_files,
debug_files,
build_utils.gn_arch_to_bazel(gn_target_cpu),
)
fuchsia_dir = build_utils.find_fuchsia_dir(build_dir)
# Symlink the root LICENSE file so the sysroot package can reference it.
build_utils.force_symlink(
repository_dir / "LICENSE", fuchsia_dir / "LICENSE"
)
module_content = dedent(
f"""\
module(name = "{repository_name}")
bazel_dep(name = "fuchsia_rules_common", version = "")
local_path_override(
module_name = "fuchsia_rules_common",
path = "{fuchsia_dir}/build/bazel_sdk/fuchsia_rules_common",
)
bazel_dep(name = "platforms", version = "1.1.0")
bazel_dep(name = "rules_license", version = "1.0.0")
"""
)
load_block = dedent(
"""\
load(
"@fuchsia_rules_common//debug_symbols:debug_symbols.bzl",
"fuchsia_unstripped_binary",
)
load("@rules_license//rules:license.bzl", "license")"""
)
dist_targets_str = "\n\n".join(dist_targets)
write_file_if_changed(repository_dir / "MODULE.bazel", module_content)
header_files_str = "".join(f' "{f}",\n' for f in header_files)
lib_files_str = "".join(f' "{f}",\n' for f in lib_files)
write_file_if_changed(
repository_dir / "BUILD.bazel",
f"""# AUTO-GENERATED - DO NOT EDIT
{load_block}
package(default_applicable_licenses = [":license"])
license(
name = "license",
license_kinds = ["@rules_license//licenses/spdx:BSD-2-Clause"],
license_text = "LICENSE",
)
exports_files(["sysroot/empty"])
filegroup(
name = "sysroot_header_files",
srcs = [
{header_files_str} ],
visibility = ["//visibility:public"]
)
filegroup(
name = "sysroot_library_files",
srcs = [
{lib_files_str} ],
visibility = ["//visibility:public"]
)
{dist_targets_str}
""",
)
def repository_name(label: str) -> str:
"""Returns repository name of the input label.
Supports both canonical repository names (starts with @@) and apparent
repository names (starts with @).
"""
repository, sep, _ = label.partition("//")
assert sep == "//", f"Missing // in label: {label}"
return repository.removeprefix("@@").removeprefix("@")
def innermost_repository_name(label: str) -> str:
"""Returns the innermost repository names.
* For top-level repositories, this is their canonical repository name.
* For repos generated by extensions, this is the repo_name used their
corresponding extensions' repo namespaces. Repo generated by extensions
have the canonical names in the form of
`module_repo_canonical_name+extension_name+repo_name`, this function
returns `repo_name` from it.
See https://bazel.build/external/extension#repository_names_and_visibility
NOTE: It's clearly stated in the doc that this naming convention is NOT an
API. However, in our use cases, we'd either do rely on this or hardcode
canonical names for extension generated repos everywhere. And the latter
is even less reliable.
"""
canonical_repo_name = repository_name(label)
if _BAZEL_REPO_NAME_SEPARATOR not in canonical_repo_name:
return canonical_repo_name
elements = canonical_repo_name.split(_BAZEL_REPO_NAME_SEPARATOR)
if len(elements) < 3:
# This is a repo_name+version canonical name, return as-is.
return canonical_repo_name
return elements[-1]