blob: c0e265b56a033c13580bbd233f089cc5307bec62 [file]
#!/usr/bin/env fuchsia-vendored-python
# Copyright 2026 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import hashlib
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
# Ensure repository root is on sys.path when running standalone.
_REPO_ROOT = Path(__file__).resolve().parents[3]
if str(_REPO_ROOT) not in sys.path:
sys.path.insert(0, str(_REPO_ROOT))
from build.bazel.scripts import generate_fx_host_tool
class GenerateFxHostToolTest(unittest.TestCase):
def test_relocated_tool_preserves_cwd_runfiles_and_runtime_tools(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
data_file = execroot / "pkg" / "greeting.txt"
data_file.parent.mkdir(parents=True)
data_file.write_text("hello from runfiles")
repo_mapping = bin_dir / "my_tool_bin.repo_mapping"
repo_mapping.write_text(",main,_main\n")
helper_tool = execroot / "prebuilt_out" / "custom-helper"
helper_tool.parent.mkdir(parents=True)
helper_tool.write_text("#!/bin/bash\necho helper-ok\n")
helper_tool.chmod(0o755)
entry_point = bin_dir / "my_tool_bin"
entry_point.write_text(
"#!/bin/bash\n"
"set -euo pipefail\n"
'echo "cwd=${PWD}"\n'
'echo "data=$(cat "${RUNFILES_DIR}/_main/pkg/greeting.txt")"\n'
'echo "helper=$(custom-helper)"\n'
'echo "args=$*"\n'
)
entry_point.chmod(0o755)
manifest_file = bin_dir / "my_tool_bin.runfiles_manifest"
manifest_file.write_text(
f"_main/__init__.py \n"
f"_main/pkg/greeting.txt pkg/greeting.txt\n"
f"_repo_mapping {repo_mapping}\n"
)
bazel_out_launcher = bin_dir / "my-tool"
bazel_out_runtime = bin_dir / "my-tool.runtime_dir"
rc = generate_fx_host_tool.generate_fx_host_tool(
entry_point=entry_point.relative_to(execroot),
entry_runfiles_manifest=manifest_file.relative_to(execroot),
output_launcher=bazel_out_launcher,
output_runtime_dir=bazel_out_runtime,
bazel_execroot=execroot,
data_runfiles=[],
runtime_tools={
"custom-helper": helper_tool.relative_to(execroot)
},
)
self.assertEqual(rc, 0)
# Simulate `fx` copying the built tool + runtime_dir into out/default/host-tools.
host_tools_dir = tmp_path / "out" / "default" / "host-tools"
host_tools_dir.mkdir(parents=True)
shutil.copy2(bazel_out_launcher, host_tools_dir / "my-tool")
shutil.copytree(
bazel_out_runtime,
host_tools_dir / "my-tool.runtime_dir",
symlinks=True,
)
launcher_text = (host_tools_dir / "my-tool").read_text()
self.assertIn(
'export FX_RUNTIME_TOOLS_DIR="${_FX_RUNTIME_DIR}/.bin"',
launcher_text,
)
caller_cwd = tmp_path / "caller_workdir"
caller_cwd.mkdir()
proc = subprocess.run(
[str(host_tools_dir / "my-tool"), "--flag", "val"],
cwd=caller_cwd,
env={"PATH": "/usr/bin:/bin"},
capture_output=True,
text=True,
check=True,
)
lines = proc.stdout.strip().splitlines()
self.assertEqual(
lines,
[
f"cwd={caller_cwd.resolve()}",
"data=hello from runfiles",
"helper=helper-ok",
"args=--flag val",
],
)
# Empty __init__.py entry must also be materialized in relocated runfiles.
self.assertTrue(
(
host_tools_dir
/ "my-tool.runtime_dir"
/ "my_tool_bin.runfiles"
/ "_main"
/ "__init__.py"
).is_file()
)
def test_single_file_executable_without_runfiles_manifest(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
entry_point = bin_dir / "plain_tool"
entry_point.write_text('#!/bin/bash\necho "plain:$1:${PWD}"\n')
entry_point.chmod(0o755)
out_launcher = bin_dir / "plain-tool"
out_runtime = bin_dir / "plain-tool.runtime_dir"
rc = generate_fx_host_tool.generate_fx_host_tool(
entry_point=entry_point.relative_to(execroot),
entry_runfiles_manifest=None,
output_launcher=out_launcher,
output_runtime_dir=out_runtime,
bazel_execroot=execroot,
data_runfiles=[],
runtime_tools={},
)
self.assertEqual(rc, 0)
host_tools_dir = tmp_path / "host-tools"
host_tools_dir.mkdir()
shutil.copy2(out_launcher, host_tools_dir / "plain-tool")
shutil.copytree(
out_runtime,
host_tools_dir / "plain-tool.runtime_dir",
symlinks=True,
)
caller_cwd = tmp_path / "sub_cwd"
caller_cwd.mkdir()
proc = subprocess.run(
[str(host_tools_dir / "plain-tool"), "ok"],
cwd=caller_cwd,
env={"PATH": os.environ.get("PATH", "/usr/bin:/bin")},
capture_output=True,
text=True,
check=True,
)
self.assertEqual(
proc.stdout.strip(), f"plain:ok:{caller_cwd.resolve()}"
)
# Compiled/generated entry points inside bazel-out remain regular files, not symlinks.
self.assertFalse((out_runtime / "plain_tool").is_symlink())
def test_in_tree_source_files_staged_for_isolated_swarming_relocation(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
build_bot_fuchsia_dir = tmp_path / "cache" / "incremental"
build_bot_out = build_bot_fuchsia_dir / "out" / "not-default"
execroot = (
build_bot_out
/ "gen"
/ "build"
/ "bazel"
/ "output_base"
/ "execroot"
/ "_main"
)
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
in_tree_src = (
build_bot_fuchsia_dir
/ "tools"
/ "devshell"
/ "host_tools"
/ "foo"
/ "foo.sh"
)
in_tree_src.parent.mkdir(parents=True)
in_tree_src.write_text(
"#!/bin/bash\n"
"set -euo pipefail\n"
'cat "${RUNFILES_DIR}/_main/tools/devshell/host_tools/foo/helper.txt"\n'
)
in_tree_src.chmod(0o755)
in_tree_helper = in_tree_src.parent / "helper.txt"
in_tree_helper.write_text("swarming-isolated-ok\n")
(execroot / "tools" / "devshell" / "host_tools" / "foo").mkdir(
parents=True
)
(
execroot
/ "tools"
/ "devshell"
/ "host_tools"
/ "foo"
/ "foo.sh"
).symlink_to(in_tree_src)
(
execroot
/ "tools"
/ "devshell"
/ "host_tools"
/ "foo"
/ "helper.txt"
).symlink_to(in_tree_helper)
out_launcher = bin_dir / "foo"
out_runtime = bin_dir / "foo.runtime_dir"
rc = generate_fx_host_tool.generate_fx_host_tool(
entry_point=Path("tools/devshell/host_tools/foo/foo.sh"),
entry_runfiles_manifest=None,
output_launcher=out_launcher,
output_runtime_dir=out_runtime,
bazel_execroot=execroot,
data_runfiles=["tools/devshell/host_tools/foo/helper.txt"],
runtime_tools={},
fuchsia_dir=build_bot_fuchsia_dir,
)
self.assertEqual(rc, 0)
self.assertFalse((out_runtime / "foo.sh").is_symlink())
self.assertFalse(
(
out_runtime
/ "foo.sh.runfiles"
/ "_main"
/ "tools"
/ "devshell"
/ "host_tools"
/ "foo"
/ "helper.txt"
).is_symlink()
)
# Simulate Swarming CAS unpacking `out/not-default` at `/b/s/w/ir/out/not-default`
# on a clean test bot where `cache/incremental` does not exist.
swarming_root = tmp_path / "swarming_ir"
swarming_out = swarming_root / "out" / "not-default"
swarming_out.parent.mkdir(parents=True)
shutil.move(str(build_bot_out), str(swarming_out))
shutil.rmtree(build_bot_fuchsia_dir)
swarming_launcher = (
swarming_out
/ "gen"
/ "build"
/ "bazel"
/ "output_base"
/ "execroot"
/ "_main"
/ "bazel-out"
/ "k8-fastbuild"
/ "bin"
/ "pkg"
/ "foo"
)
proc = subprocess.run(
[str(swarming_launcher)],
env={"PATH": "/usr/bin:/bin"},
capture_output=True,
text=True,
check=True,
)
self.assertEqual(proc.stdout.strip(), "swarming-isolated-ok")
def test_always_build_flag_writes_marker_file(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
entry_point = bin_dir / "compiled_bin"
entry_point.write_text("#!/bin/bash\nexit 0\n")
entry_point.chmod(0o755)
no_marker_runtime = bin_dir / "no-marker.runtime_dir"
rc1 = generate_fx_host_tool.main(
[
f"--entry-point={entry_point.relative_to(execroot)}",
f"--output-launcher={bin_dir / 'no-marker'}",
f"--output-runtime-dir={no_marker_runtime}",
f"--bazel-execroot={execroot}",
]
)
self.assertEqual(rc1, 0)
self.assertFalse((no_marker_runtime / ".always_build").exists())
marker_runtime = bin_dir / "with-marker.runtime_dir"
rc2 = generate_fx_host_tool.main(
[
f"--entry-point={entry_point.relative_to(execroot)}",
f"--output-launcher={bin_dir / 'with-marker'}",
f"--output-runtime-dir={marker_runtime}",
f"--bazel-execroot={execroot}",
"--always-build",
]
)
self.assertEqual(rc2, 0)
self.assertTrue((marker_runtime / ".always_build").is_file())
def test_runtime_fx_tool_composition_survives_relocation_and_source_deletion(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
child_bin_dir = (
execroot / "bazel-out" / "k8-fastbuild" / "bin" / "child_pkg"
)
parent_bin_dir = (
execroot / "bazel-out" / "k8-fastbuild" / "bin" / "parent_pkg"
)
child_bin_dir.mkdir(parents=True)
parent_bin_dir.mkdir(parents=True)
inner_helper = execroot / "prebuilt_out" / "inner-cipd-bin"
inner_helper.parent.mkdir(parents=True)
inner_helper.write_text("#!/bin/bash\necho inner-cipd-ok\n")
inner_helper.chmod(0o755)
child_data = execroot / "child_pkg" / "config.txt"
child_data.parent.mkdir(parents=True)
child_data.write_text("child-runfile-ok")
child_entry = child_bin_dir / "child_entry_bin"
child_entry.write_text(
"#!/bin/bash\n"
"set -euo pipefail\n"
'echo "child_fx_tools_dir_has_inner=$([[ -x "${FX_RUNTIME_TOOLS_DIR}/inner-cipd-bin" ]] && echo yes || echo no)"\n'
'echo "child_helper=$(inner-cipd-bin)"\n'
'echo "child_data=$(cat "${RUNFILES_DIR}/_main/child_pkg/config.txt")"\n'
'echo "child_args=$*"\n'
)
child_entry.chmod(0o755)
child_launcher = child_bin_dir / "child-tool"
child_runtime = child_bin_dir / "child-tool.runtime_dir"
rc_child = generate_fx_host_tool.main(
[
f"--entry-point={child_entry.relative_to(execroot)}",
f"--output-launcher={child_launcher}",
f"--output-runtime-dir={child_runtime}",
f"--bazel-execroot={execroot}",
"--data-runfile=child_pkg/config.txt",
f"--runtime-tool=inner-cipd-bin={inner_helper.relative_to(execroot)}",
]
)
self.assertEqual(rc_child, 0)
parent_entry = (
execroot / "tools" / "parent_pkg" / "parent_script.sh"
)
parent_entry.parent.mkdir(parents=True)
parent_entry.write_text(
"#!/bin/bash\n"
"set -euo pipefail\n"
'echo "fx_tools_dir_has_child=$([[ -x "${FX_RUNTIME_TOOLS_DIR}/child-tool" ]] && echo yes || echo no)"\n'
'child-tool "$@"\n'
)
parent_entry.chmod(0o755)
parent_launcher = parent_bin_dir / "parent-tool"
parent_runtime = parent_bin_dir / "parent-tool.runtime_dir"
rc_parent = generate_fx_host_tool.main(
[
f"--entry-point={parent_entry.relative_to(execroot)}",
f"--output-launcher={parent_launcher}",
f"--output-runtime-dir={parent_runtime}",
f"--bazel-execroot={execroot}",
(
"--runtime-fx-tool=child-tool="
f"{child_launcher.relative_to(execroot)}:"
f"{child_runtime.relative_to(execroot)}"
),
]
)
self.assertEqual(rc_parent, 0)
staged_child_launcher = parent_runtime / ".bin" / "child-tool"
staged_child_runtime = (
parent_runtime / ".bin" / "child-tool.runtime_dir"
)
self.assertTrue(staged_child_launcher.is_file())
self.assertFalse(staged_child_launcher.is_symlink())
self.assertTrue(os.access(staged_child_launcher, os.X_OK))
self.assertTrue(staged_child_runtime.is_dir())
self.assertFalse(staged_child_runtime.is_symlink())
# Relocate parent-tool to host-tools/ and DELETE child_bin_dir completely.
host_tools_dir = tmp_path / "host-tools"
host_tools_dir.mkdir()
shutil.copy2(parent_launcher, host_tools_dir / "parent-tool")
shutil.copytree(
parent_runtime,
host_tools_dir / "parent-tool.runtime_dir",
symlinks=True,
)
shutil.rmtree(child_bin_dir)
proc = subprocess.run(
[str(host_tools_dir / "parent-tool"), "--via-parent"],
env={"PATH": "/usr/bin:/bin"},
capture_output=True,
text=True,
check=True,
)
self.assertEqual(
proc.stdout.strip().splitlines(),
[
"fx_tools_dir_has_child=yes",
"child_fx_tools_dir_has_inner=yes",
"child_helper=inner-cipd-ok",
"child_data=child-runfile-ok",
"child_args=--via-parent",
],
)
def test_directory_runtime_tool_is_rejected(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
clang_tree = (
execroot
/ "bazel-out"
/ "k8-fastbuild"
/ "bin"
/ "cipd"
/ "prebuilt"
/ "third_party"
/ "clang"
/ "linux-x64"
)
(clang_tree / "bin").mkdir(parents=True)
clang_exe = clang_tree / "bin" / "clang"
clang_exe.write_text("#!/bin/bash\necho clang-from-pkg-root\n")
clang_exe.chmod(0o755)
entry_point = bin_dir / "wrapper.sh"
entry_point.write_text('#!/bin/bash\nexec clang "$@"\n')
entry_point.chmod(0o755)
out_launcher = bin_dir / "llvm-wrap"
out_runtime = bin_dir / "llvm-wrap.runtime_dir"
rc = generate_fx_host_tool.generate_fx_host_tool(
entry_point=entry_point.relative_to(execroot),
entry_runfiles_manifest=None,
output_launcher=out_launcher,
output_runtime_dir=out_runtime,
bazel_execroot=execroot,
data_runfiles=[],
runtime_tools={
"linux-x64": clang_tree.relative_to(execroot),
},
)
self.assertEqual(rc, 1)
def test_generated_data_runfiles_map_short_path_to_bazel_out_path(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
gen_shac = (
execroot
/ "bazel-out"
/ "k8-fastbuild"
/ "bin"
/ "cipd"
/ "prebuilt"
/ "tools"
/ "shac"
/ "shac"
)
gen_shac.parent.mkdir(parents=True)
gen_shac.write_text("#!/bin/bash\necho real-shac\n")
gen_shac.chmod(0o755)
entry_point = bin_dir / "runner"
entry_point.write_text("#!/bin/bash\nexit 0\n")
entry_point.chmod(0o755)
out_launcher = bin_dir / "fmt-tool"
out_runtime = bin_dir / "fmt-tool.runtime_dir"
short_path = "cipd/prebuilt/tools/shac/shac"
exec_path = str(gen_shac.relative_to(execroot))
rc = generate_fx_host_tool.generate_fx_host_tool(
entry_point=entry_point.relative_to(execroot),
entry_runfiles_manifest=None,
output_launcher=out_launcher,
output_runtime_dir=out_runtime,
bazel_execroot=execroot,
data_runfiles=[f"{short_path}={exec_path}"],
runtime_tools={},
)
self.assertEqual(rc, 0)
staged = (
out_runtime / "runner.runfiles" / "_main" / Path(short_path)
)
self.assertTrue(staged.is_file())
self.assertFalse(staged.is_symlink())
self.assertEqual(staged.read_text(), gen_shac.read_text())
def test_survives_bazel_clean_with_bzlmod_repo_mapping_and_cipd_dir(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
fuchsia_dir = tmp_path / "fuchsia"
output_base = (
fuchsia_dir
/ "out"
/ "default"
/ "gen"
/ "build"
/ "bazel"
/ "output_base"
)
execroot = output_base / "execroot" / "_main"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
# In-tree source file under $FUCHSIA_DIR (symlinked in execroot).
real_src_data = fuchsia_dir / "pkg" / "in_tree.txt"
real_src_data.parent.mkdir(parents=True)
real_src_data.write_text("in-tree-v1")
(execroot / "pkg").mkdir(parents=True)
(execroot / "pkg" / "in_tree.txt").symlink_to(real_src_data)
# External repository file under output_base/external/rules_python+/...
ext_file = (
output_base
/ "external"
/ "rules_python+"
/ "python"
/ "runfiles"
/ "runfiles.py"
)
ext_file.parent.mkdir(parents=True)
ext_file.write_text("# external runfiles helper\n")
# CIPD directory under $FUCHSIA_DIR/prebuilt/third_party/clang/linux-x64
# with multicall clang++ -> clang -> llvm symlink chain and ../lib/libclang.so.
clang_src_dir = (
fuchsia_dir / "prebuilt" / "third_party" / "clang" / "linux-x64"
)
(clang_src_dir / "bin").mkdir(parents=True)
(clang_src_dir / "lib").mkdir(parents=True)
llvm_multicall = clang_src_dir / "bin" / "llvm"
llvm_multicall.write_text(
"#!/bin/bash\n"
'echo "argv0=$(basename "$0")"\n'
'_exe_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"\n'
'echo "has_lib=$([[ -f "${_exe_dir}/../lib/libclang.so" ]] && echo yes || echo no)"\n'
)
llvm_multicall.chmod(0o755)
(clang_src_dir / "bin" / "clang").symlink_to("llvm")
(clang_src_dir / "bin" / "clang++").symlink_to("clang")
(clang_src_dir / "lib" / "libclang.so").write_text("libclang")
# TreeArtifact copy in bazel-out (which will be deleted on `fx bazel clean`).
clang_tree = (
bin_dir / "prebuilt" / "third_party" / "clang" / "linux-x64"
)
shutil.copytree(clang_src_dir, clang_tree, symlinks=True)
# Single-file CIPD / wrapper tool in bazel-out with auxiliary wrapped_bin in runfiles.
wrapper_src = fuchsia_dir / "tools" / "cipd" / "shac_runner.sh"
wrapper_src.parent.mkdir(parents=True)
wrapper_src.write_text(
"#!/bin/bash\n"
"set -euo pipefail\n"
'for rf in "${FX_RUNTIME_TOOLS_DIR}/.."/*.runfiles "${BASH_SOURCE[0]}.runfiles"; do\n'
' if [[ -x "${rf}/_main/tools/cipd/shac.wrapped_bin" ]]; then\n'
' exec "${rf}/_main/tools/cipd/shac.wrapped_bin" "$@"\n'
" fi\n"
"done\n"
"exit 1\n"
)
wrapper_src.chmod(0o755)
wrapper_out = bin_dir / "shac"
wrapper_out.symlink_to(wrapper_src)
wrapped_bin = bin_dir / "shac.wrapped_bin"
wrapped_bin.write_text("#!/bin/bash\necho shac-wrapped-ok:$*\n")
wrapped_bin.chmod(0o755)
repo_mapping = bin_dir / "tool_bin.repo_mapping"
repo_mapping.write_text(
",fuchsia,_main\n,rules_python,rules_python+\n"
)
gen_so = bin_dir / "libhelper.so"
gen_so.write_text("fake-so")
entry_point = bin_dir / "tool_bin"
entry_point.write_text(
"#!/bin/bash\n"
"set -euo pipefail\n"
'echo "manifest_env=${RUNFILES_MANIFEST_FILE:-UNSET}"\n'
'echo "ld_env=${LD_LIBRARY_PATH:-UNSET}"\n'
'echo "repo_map=$(head -n 1 "${RUNFILES_DIR}/_repo_mapping")"\n'
'echo "in_tree=$(cat "${RUNFILES_DIR}/_main/pkg/in_tree.txt")"\n'
'echo "ext_exists=$([[ -f "${RUNFILES_DIR}/rules_python+/python/runfiles/runfiles.py" ]] && echo yes || echo no)"\n'
'echo "legacy_ext_removed=$([[ -e "${RUNFILES_DIR}/_main/external/rules_python+/python/runfiles/runfiles.py" ]] && echo no || echo yes)"\n'
"clang++\n"
"shac check\n"
)
entry_point.chmod(0o755)
manifest_file = bin_dir / "tool_bin.runfiles_manifest"
manifest_file.write_text(
f"_main/pkg/in_tree.txt {execroot / 'pkg' / 'in_tree.txt'}\n"
f"_main/pkg/libhelper.so {gen_so}\n"
f"rules_python+/python/runfiles/runfiles.py {ext_file}\n"
f"_main/external/rules_python+/python/runfiles/runfiles.py {ext_file}\n"
f"_repo_mapping {repo_mapping}\n"
)
out_launcher = bin_dir / "clean-safe-tool"
out_runtime = bin_dir / "clean-safe-tool.runtime_dir"
rc = generate_fx_host_tool.main(
[
f"--entry-point={entry_point.relative_to(execroot)}",
f"--entry-runfiles-manifest={manifest_file.relative_to(execroot)}",
f"--output-launcher={out_launcher}",
f"--output-runtime-dir={out_runtime}",
f"--bazel-execroot={execroot}",
f"--fuchsia-dir={fuchsia_dir}",
f"--data-runfile=tools/cipd/shac.wrapped_bin={wrapped_bin.relative_to(execroot)}",
f"--runtime-tool=shac={wrapper_out.relative_to(execroot)}",
(
"--runtime-cipd-dir=linux-x64="
f"{clang_tree.relative_to(execroot)}:"
+ "/".join(
("prebuilt", "third_party", "clang", "linux-x64")
)
),
]
)
self.assertEqual(rc, 0)
launcher_text = out_launcher.read_text()
self.assertNotIn("LD_LIBRARY_PATH", launcher_text)
self.assertIn("unset RUNFILES_MANIFEST_FILE", launcher_text)
# Relocate into out/default/host-tools and simulate `fx bazel clean --expunge`
# by deleting the entire `output_base` directory tree.
host_tools_dir = fuchsia_dir / "out" / "default" / "host-tools"
host_tools_dir.mkdir(parents=True)
shutil.copy2(out_launcher, host_tools_dir / "clean-safe-tool")
shutil.copytree(
out_runtime,
host_tools_dir / "clean-safe-tool.runtime_dir",
symlinks=True,
)
shutil.rmtree(output_base)
# Deleting or mutating the original source file leaves the staged copy self-contained.
real_src_data.unlink()
# Manifest entries inside relocated runtime_dir must be relative, not pointing to deleted output_base.
relocated_manifest = (
host_tools_dir
/ "clean-safe-tool.runtime_dir"
/ "tool_bin.runfiles"
/ "MANIFEST"
).read_text()
self.assertNotIn(str(output_base), relocated_manifest)
# Even if caller environment has a stale RUNFILES_MANIFEST_FILE, launcher unsets it.
caller_cwd = tmp_path / "arbitrary_cwd"
caller_cwd.mkdir()
proc = subprocess.run(
[str(host_tools_dir / "clean-safe-tool")],
cwd=caller_cwd,
env={
"PATH": "/usr/bin:/bin",
"RUNFILES_MANIFEST_FILE": "/nonexistent/MANIFEST",
},
capture_output=True,
text=True,
check=False,
)
self.assertEqual(proc.returncode, 0, msg=proc.stderr)
self.assertEqual(
proc.stdout.strip().splitlines(),
[
"manifest_env=UNSET",
"ld_env=UNSET",
"repo_map=,fuchsia,_main",
"in_tree=in-tree-v1",
"ext_exists=yes",
"legacy_ext_removed=yes",
"argv0=clang++",
"has_lib=yes",
"shac-wrapped-ok:check",
],
)
def test_cipd_prebuilt_dir_in_data_runfiles_symlinks_to_persistent_prebuilt(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
fuchsia_dir = tmp_path / "fuchsia"
output_base = (
fuchsia_dir
/ "out"
/ "default"
/ "gen"
/ "build"
/ "bazel"
/ "output_base"
)
execroot = output_base / "execroot" / "_main"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
prebuilt_rel = Path("prebuilt", "third_party", "go", "linux-x64")
go_src_dir = fuchsia_dir / prebuilt_rel
(go_src_dir / "bin").mkdir(parents=True)
go_bin = go_src_dir / "bin" / "go"
go_bin.write_text("#!/bin/bash\necho go-ok\n")
go_bin.chmod(0o755)
(go_src_dir / "bin" / "gofmt").symlink_to("go")
# Ephemeral TreeArtifact produced by cipd_prebuilt_directory in bazel-out.
go_tree = (
execroot
/ "bazel-out"
/ "k8-fastbuild"
/ "bin"
/ "tools"
/ "devshell"
/ "host_tools"
/ "cipd"
/ prebuilt_rel
)
shutil.copytree(go_src_dir, go_tree, symlinks=True)
entry_point = bin_dir / "runner_bin"
entry_point.write_text("#!/bin/bash\nexit 0\n")
entry_point.chmod(0o755)
short_path = f"tools/devshell/host_tools/cipd/{prebuilt_rel}"
exec_path = str(go_tree.relative_to(execroot))
out_launcher = bin_dir / "go-data-tool"
out_runtime = bin_dir / "go-data-tool.runtime_dir"
rc = generate_fx_host_tool.main(
[
f"--entry-point={entry_point.relative_to(execroot)}",
f"--output-launcher={out_launcher}",
f"--output-runtime-dir={out_runtime}",
f"--bazel-execroot={execroot}",
f"--fuchsia-dir={fuchsia_dir}",
f"--data-runfile={short_path}={exec_path}",
]
)
self.assertEqual(rc, 0)
staged_go_dir = (
out_runtime / "runner_bin.runfiles" / "_main" / Path(short_path)
)
self.assertTrue(staged_go_dir.is_symlink())
self.assertEqual(staged_go_dir.resolve(), go_src_dir.resolve())
# Deleting ephemeral output_base leaves the staged runfiles symlink intact.
host_tools_dir = fuchsia_dir / "out" / "default" / "host-tools"
host_tools_dir.mkdir(parents=True)
shutil.copytree(
out_runtime,
host_tools_dir / "go-data-tool.runtime_dir",
symlinks=True,
)
shutil.rmtree(output_base)
relocated_go_dir = (
host_tools_dir
/ "go-data-tool.runtime_dir"
/ "runner_bin.runfiles"
/ "_main"
/ Path(short_path)
)
self.assertTrue(relocated_go_dir.is_dir())
self.assertTrue((relocated_go_dir / "bin" / "gofmt").is_symlink())
def test_ephemeral_directory_in_data_runfiles_preserves_internal_symlinks(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
output_base = tmp_path / "output_base"
execroot = output_base / "execroot" / "_main"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
gen_bundle = bin_dir / "gen_bundle_dir"
(gen_bundle / "bin").mkdir(parents=True)
multicall = gen_bundle / "bin" / "multicall"
multicall.write_text('#!/bin/bash\necho "mc:$(basename "$0")"\n')
multicall.chmod(0o755)
(gen_bundle / "bin" / "sub1").symlink_to("multicall")
(gen_bundle / "bin" / "sub2").symlink_to("sub1")
entry_point = bin_dir / "bundle_runner"
entry_point.write_text("#!/bin/bash\nexit 0\n")
entry_point.chmod(0o755)
out_launcher = bin_dir / "bundle-tool"
out_runtime = bin_dir / "bundle-tool.runtime_dir"
rc = generate_fx_host_tool.main(
[
f"--entry-point={entry_point.relative_to(execroot)}",
f"--output-launcher={out_launcher}",
f"--output-runtime-dir={out_runtime}",
f"--bazel-execroot={execroot}",
f"--data-runfile=pkg/gen_bundle_dir={gen_bundle.relative_to(execroot)}",
]
)
self.assertEqual(rc, 0)
host_tools_dir = tmp_path / "host-tools"
host_tools_dir.mkdir()
shutil.copytree(
out_runtime,
host_tools_dir / "bundle-tool.runtime_dir",
symlinks=True,
)
shutil.rmtree(output_base)
staged_bundle = (
host_tools_dir
/ "bundle-tool.runtime_dir"
/ "bundle_runner.runfiles"
/ "_main"
/ "pkg"
/ "gen_bundle_dir"
)
self.assertTrue(staged_bundle.is_dir())
self.assertFalse(staged_bundle.is_symlink())
self.assertTrue((staged_bundle / "bin" / "sub1").is_symlink())
self.assertEqual(
os.readlink(staged_bundle / "bin" / "sub1"), "multicall"
)
self.assertTrue((staged_bundle / "bin" / "sub2").is_symlink())
self.assertEqual(
os.readlink(staged_bundle / "bin" / "sub2"), "sub1"
)
def test_sources_manifest_and_build_working_directory(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
source_file = execroot / "tools" / "tool.py"
source_file.parent.mkdir(parents=True)
source_file.write_text("print('hello')\n", encoding="utf-8")
runner_src = (
execroot / "build" / "prebuilt" / "cipd" / "shac_runner.sh"
)
runner_src.parent.mkdir(parents=True)
runner_src.write_text("#!/bin/bash\nexit 0\n", encoding="utf-8")
runner_src.chmod(0o644)
gen_symlink = bin_dir / "shac"
gen_symlink.symlink_to(runner_src)
prebuilt_rel = Path("prebuilt", "third_party", "foo", "bar.txt")
prebuilt_file = execroot / prebuilt_rel
prebuilt_file.parent.mkdir(parents=True)
prebuilt_file.write_text("prebuilt", encoding="utf-8")
entry_point = bin_dir / "compiled_bin"
entry_point.write_text("#!/bin/bash\nexit 0\n")
entry_point.chmod(0o755)
out_launcher = bin_dir / "my-tool"
out_runtime = bin_dir / "my-tool.runtime_dir"
rc = generate_fx_host_tool.main(
[
f"--entry-point={entry_point.relative_to(execroot)}",
f"--output-launcher={out_launcher}",
f"--output-runtime-dir={out_runtime}",
f"--bazel-execroot={execroot}",
f"--source-file=tools/tool.py={source_file.relative_to(execroot)}",
f"--source-file=build/prebuilt/cipd/shac_runner.sh={runner_src.relative_to(execroot)}",
f"--data-runfile=build/prebuilt/cipd/shac={gen_symlink.relative_to(execroot)}",
f"--data-runfile={prebuilt_rel}={prebuilt_file.relative_to(execroot)}",
f"--runtime-tool=shac={gen_symlink.relative_to(execroot)}",
]
)
self.assertEqual(rc, 0)
manifest = out_runtime / "sources.manifest"
self.assertTrue(manifest.is_file())
content = manifest.read_text(encoding="utf-8")
expected_git_sha = hashlib.sha1(
b"blob 15\0print('hello')\n"
).hexdigest()
self.assertIn(f"{expected_git_sha} tools/tool.py\n", content)
self.assertIn(" build/prebuilt/cipd/shac_runner.sh\n", content)
self.assertNotIn(" build/prebuilt/cipd/shac\n", content)
self.assertNotIn(str(prebuilt_rel), content)
self.assertTrue(os.access(out_runtime / ".bin" / "shac", os.X_OK))
launcher_content = out_launcher.read_text(encoding="utf-8")
self.assertIn("BUILD_WORKING_DIRECTORY", launcher_content)
self.assertIn('cd "${BUILD_WORKING_DIRECTORY}"', launcher_content)
self.assertIn("unset BUILD_WORKING_DIRECTORY", launcher_content)
def test_launcher_with_relative_invocation_and_build_working_directory(
self,
) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
bin_dir = execroot / "bazel-out" / "bin"
bin_dir.mkdir(parents=True)
entry_point = bin_dir / "tool_bin"
entry_point.write_text(
"#!/bin/bash\n"
"set -euo pipefail\n"
'echo "cwd=${PWD}"\n'
'echo "bwd=${BUILD_WORKING_DIRECTORY:-UNSET}"\n'
'echo "args=$*"\n'
)
entry_point.chmod(0o755)
out_launcher = bin_dir / "my-tool"
out_runtime = bin_dir / "my-tool.runtime_dir"
rc = generate_fx_host_tool.generate_fx_host_tool(
entry_point=entry_point.relative_to(execroot),
entry_runfiles_manifest=None,
output_launcher=out_launcher,
output_runtime_dir=out_runtime,
bazel_execroot=execroot,
data_runfiles=[],
runtime_tools={},
)
self.assertEqual(rc, 0)
launcher_lines = out_launcher.read_text(
encoding="utf-8"
).splitlines()
tool_dir_idx = next(
i
for i, line in enumerate(launcher_lines)
if "_FX_TOOL_DIR=" in line
)
cd_idx = next(
i
for i, line in enumerate(launcher_lines)
if 'cd "${BUILD_WORKING_DIRECTORY}"' in line
)
unset_bwd_idx = next(
i
for i, line in enumerate(launcher_lines)
if "unset BUILD_WORKING_DIRECTORY" in line
)
self.assertLess(
tool_dir_idx,
cd_idx,
msg="_FX_TOOL_DIR must be resolved before changing into BUILD_WORKING_DIRECTORY",
)
self.assertLess(
cd_idx,
unset_bwd_idx,
msg="unset BUILD_WORKING_DIRECTORY must follow cd ${BUILD_WORKING_DIRECTORY}",
)
work_dir = tmp_path / "other_work_dir"
work_dir.mkdir()
# Execute launcher via relative path `./my-tool` from bin_dir,
# with BUILD_WORKING_DIRECTORY pointing to work_dir.
proc = subprocess.run(
["./my-tool", "hello", "world"],
cwd=bin_dir,
env={
"PATH": "/usr/bin:/bin",
"BUILD_WORKING_DIRECTORY": str(work_dir),
},
capture_output=True,
text=True,
check=True,
)
lines = proc.stdout.strip().splitlines()
self.assertEqual(lines[0], f"cwd={work_dir.resolve()}")
self.assertEqual(lines[1], "bwd=UNSET")
self.assertEqual(lines[2], "args=hello world")
def test_packages_manifest_generation(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
execroot = tmp_path / "execroot"
bin_dir = execroot / "bazel-out" / "k8-fastbuild" / "bin" / "pkg"
bin_dir.mkdir(parents=True)
entry_point = bin_dir / "compiled_bin"
entry_point.write_text("#!/bin/bash\nexit 0\n")
entry_point.chmod(0o755)
out_launcher = bin_dir / "my-tool"
out_runtime = bin_dir / "my-tool.runtime_dir"
rc = generate_fx_host_tool.main(
[
f"--entry-point={entry_point.relative_to(execroot)}",
f"--output-launcher={out_launcher}",
f"--output-runtime-dir={out_runtime}",
f"--bazel-execroot={execroot}",
"--source-package=tools/cmc",
"--source-package=tools/lib/cml",
"--source-package=",
]
)
self.assertEqual(rc, 0)
pkg_manifest = out_runtime / "packages.manifest"
self.assertTrue(pkg_manifest.is_file())
content = pkg_manifest.read_text(encoding="utf-8")
self.assertEqual(
content,
"BUILD.bazel\ntools/cmc/BUILD.bazel\ntools/lib/cml/BUILD.bazel\n",
)
if __name__ == "__main__":
unittest.main()