blob: 92d7f8784a65136539239c99958c4065a5f57d13 [file]
#!/usr/bin/env fuchsia-vendored-python
# Copyright 2024 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Unit-tests for remote_services_utils.py functions."""
import tempfile
import typing as T
import unittest
from pathlib import Path
from build.bazel.scripts import remote_services_utils
from build.rbe import rbe_profile, rbe_settings
from build.resultstore import rs_profile
class RemoteServicesUtilsTest(unittest.TestCase):
def setUp(self) -> None:
self._td = tempfile.TemporaryDirectory()
self._dir = Path(self._td.name)
self.source_dir = self._dir / "source"
self.source_dir.mkdir()
self.cfg_file1 = (
self.source_dir / remote_services_utils.RBE_CONFIG_FILES[0]
)
self.cfg_file2 = (
self.source_dir / remote_services_utils.RBE_CONFIG_FILES[1]
)
self.cfg_file1.parent.mkdir(parents=True)
self.cfg_file1.write_text(
"\n" + "# foo=bar\n" # empty line # comment line should be ignored
"instance=projects/test_project_name/instances/test_instance_name\n"
)
self.cfg_file2.parent.mkdir(exist_ok=True)
self.cfg_file2.write_text(
"platform=container-image=docker://gcr.io/test_project/debian11,something_else\n"
)
# Write fake RBE and ResultStore profiles inside the temporary source dir
rbe_config = self.source_dir / ".fx/config/rbe_profile"
rbe_config.parent.mkdir(parents=True, exist_ok=True)
rbe_config.write_text("default")
rbe_profile_json = self.source_dir / "build/rbe/profiles/default.json"
rbe_profile_json.parent.mkdir(parents=True, exist_ok=True)
rbe_profile_json.write_text(
"""{
"profile_name": "default",
"rbe_instance": "projects/test_project_name/instances/test_instance_name",
"remote_exec_root": "/fake/exec/root",
"container_image": "docker://gcr.io/fake/image",
"os_family": "fake-os",
"worker_pools": {
"default": "Pool=fake-default-pool",
"large": "Pool=fake-large-pool"
},
"auth_type": "google"
}"""
)
rs_profile_json = (
self.source_dir / "build/resultstore/profiles/default.json"
)
rs_profile_json.parent.mkdir(parents=True, exist_ok=True)
rs_profile_json.write_text(
"""{
"profile_name": "default",
"rs_instance": "projects/test_rs_project/instances/test_rs_instance",
"cas_instance": "projects/test_cas_project/instances/test_cas_instance",
"results_url": "http://fake-results-url",
"auth_type": "google"
}"""
)
def tearDown(self) -> None:
self._td.cleanup()
def test_generate_rbe_config(self) -> None:
config_dict, input_files = remote_services_utils.generate_rbe_config(
self.source_dir, environ={}
)
self.assertDictEqual(
config_dict,
{
"instance": "projects/test_project_name/instances/test_instance_name",
"rbe_instance": "projects/test_project_name/instances/test_instance_name",
"container_image": "docker://gcr.io/fake/image",
"rs_instance": "projects/test_rs_project/instances/test_rs_instance",
"rs_project": "test_rs_project",
"cas_instance": "projects/test_cas_project/instances/test_cas_instance",
"results_url": "http://fake-results-url",
"platform": "container-image=docker://gcr.io/test_project/debian11,something_else",
},
)
def test_generate_rbe_config_with_env_bypass(self) -> None:
# Simulate Option C production path using explicit environment dictionary
mock_env = {
"RBE_instance": "projects/env-project/instances/env-rbe",
"FX_BUILD_INTERNAL_RBE_REMOTE_EXEC_ROOT": "/env/exec/root",
"FX_BUILD_INTERNAL_RBE_CONTAINER_IMAGE": "docker://gcr.io/fake/image",
"FX_BUILD_INTERNAL_RBE_OS_FAMILY": "fake-os",
"FX_BUILD_INTERNAL_RBE_WORKER_POOLS": '{"default": "Pool=env-default", "large": "Pool=env-large"}',
"RS_rs_instance": "projects/env-project/instances/env-rs",
"RS_cas_instance": "projects/env-project/instances/env-cas",
"RS_results_url": "http://env-results/",
}
(
config_dict,
input_files,
) = remote_services_utils.generate_rbe_config(
self.source_dir, environ=mock_env
)
self.assertDictEqual(
config_dict,
{
"instance": "projects/test_project_name/instances/test_instance_name",
"rbe_instance": "projects/env-project/instances/env-rbe",
"container_image": "docker://gcr.io/fake/image",
"rs_instance": "projects/env-project/instances/env-rs",
"rs_project": "env-project",
"cas_instance": "projects/env-project/instances/env-cas",
"results_url": "http://env-results",
"platform": "container-image=docker://gcr.io/test_project/debian11,something_else",
},
)
def test_generate_rbe_template_substitutions(self) -> None:
_TEST_CASES: list[dict[str, T.Any]] = [
{
"name": "simple_project",
"config_dict": {
"rbe_instance": "projects/test_project1/instances/1",
"rs_instance": "projects/test_rs_project/instances/1",
"rs_project": "test_rs_project",
"cas_instance": "projects/test_cas_project/instances/1",
"results_url": "http://fake-results-url",
"platform": "foo=bar,container-image=docker://something/else,zoo=qux",
},
"remote_download_outputs": "all",
"expected": {
"remote_download_outputs": "all",
"remote_instance_name": "projects/test_project1/instances/1",
"rs_instance": "projects/test_rs_project/instances/1",
"rs_project": "test_rs_project",
"cas_instance": "projects/test_cas_project/instances/1",
"results_url": "http://fake-results-url",
"container_image": "docker://something/else",
},
},
]
for test_case in _TEST_CASES:
msg = f'For {test_case["name"]} case'
result = remote_services_utils.generate_rbe_template_substitutions(
test_case["config_dict"], test_case["remote_download_outputs"]
)
self.assertDictEqual(result, test_case["expected"], msg=msg)
def test_generate_remote_services_bazelrc(self) -> None:
build_dir = self._dir / "my_build_dir"
build_dir.mkdir(parents=True, exist_ok=True)
output_file = (
build_dir / "regenerator_outputs" / "remote_services.bazelrc"
)
output_file.parent.mkdir(parents=True, exist_ok=True)
fake_settings = rbe_settings.fake(
bazel_enable=True,
cxx_enable=True,
link_enable=True,
rust_enable=True,
needs_reproxy=True,
needs_auth=True,
)
template_file = (
self.source_dir / remote_services_utils.RBE_TEMPLATE_FILE
)
template_file.parent.mkdir(parents=True, exist_ok=True)
template_file.write_text(
r"""# AUTO-GENERATED
my_download_outputs={remote_download_outputs}
my_remote_instance_name={remote_instance_name}
my_rs_instance={rs_instance}
my_rs_project={rs_project}
my_cas_instance={cas_instance}
my_results_url={results_url}
my_container_image={container_image}
"""
)
environ: dict[str, str] = {}
input_files = remote_services_utils.generate_remote_services_bazelrc(
fuchsia_dir=self.source_dir,
build_dir=build_dir,
output_path=output_file,
download_outputs="find-me",
environ=environ,
settings=fake_settings,
)
self.assertEqual(
output_file.read_text(),
"""# AUTO-GENERATED
my_download_outputs=find-me
my_remote_instance_name=projects/test_project_name/instances/test_instance_name
my_rs_instance=projects/test_rs_project/instances/test_rs_instance
my_rs_project=test_rs_project
my_cas_instance=projects/test_cas_project/instances/test_cas_instance
my_results_url=http://fake-results-url
my_container_image=docker://gcr.io/fake/image
""",
)
# Resolve the active profile path objects expected in input_files
rbe_paths = rbe_profile.resolve_paths(self.source_dir, environ)
rs_paths = rs_profile.resolve_paths(self.source_dir, environ)
expected_inputs = {
template_file,
self.cfg_file1,
self.cfg_file2,
*rbe_paths.active_inputs,
*rs_paths.active_inputs,
}
self.assertSetEqual(input_files, expected_inputs)
def test_generate_remote_services_bazelrc_environ_profile_override(
self,
) -> None:
build_dir = self._dir / "my_build_dir"
build_dir.mkdir(parents=True, exist_ok=True)
output_file = (
build_dir / "regenerator_outputs" / "remote_services.bazelrc"
)
output_file.parent.mkdir(parents=True, exist_ok=True)
fake_settings = rbe_settings.fake(
bazel_enable=True,
cxx_enable=True,
link_enable=True,
rust_enable=True,
needs_reproxy=True,
needs_auth=True,
)
template_file = (
self.source_dir / remote_services_utils.RBE_TEMPLATE_FILE
)
template_file.parent.mkdir(parents=True, exist_ok=True)
template_file.write_text("my_results_url={results_url}\n")
# Create distinct custom profile files for RBE and ResultStore disambiguation
custom_rbe_json = self.source_dir / "build/rbe/profiles/custom_rbe.json"
custom_rbe_json.parent.mkdir(parents=True, exist_ok=True)
custom_rbe_json.write_text(
"""{
"profile_name": "custom_rbe",
"rbe_instance": "projects/custom-rbe/instances/default",
"remote_exec_root": "/fake",
"container_image": "docker://gcr.io/fake/image",
"os_family": "fake-os",
"worker_pools": {
"default": "Pool=custom-pool",
"large": "Pool=custom-large"
},
"auth_type": "google"
}"""
)
custom_rs_json = (
self.source_dir / "build/resultstore/profiles/custom_rs.json"
)
custom_rs_json.parent.mkdir(parents=True, exist_ok=True)
custom_rs_json.write_text(
"""{
"profile_name": "custom_rs",
"rs_instance": "projects/custom-rs/instances/default",
"cas_instance": "projects/custom-rs/instances/default",
"results_url": "http://custom-rs-results",
"auth_type": "google"
}"""
)
env_override = {
"FX_BUILD_RBE_PROFILE": "custom_rbe",
"FX_BUILD_RS_PROFILE": "custom_rs",
}
input_files = remote_services_utils.generate_remote_services_bazelrc(
fuchsia_dir=self.source_dir,
build_dir=build_dir,
output_path=output_file,
download_outputs="all",
environ=env_override,
settings=fake_settings,
)
rbe_paths = rbe_profile.resolve_paths(self.source_dir, env_override)
rs_paths = rs_profile.resolve_paths(self.source_dir, env_override)
for path in rbe_paths.active_inputs:
self.assertIn(path, input_files)
for path in rs_paths.active_inputs:
self.assertIn(path, input_files)
self.assertIn(custom_rbe_json, input_files)
self.assertIn(custom_rs_json, input_files)
def test_generate_remote_services_bazelrc_rbe_disabled(self) -> None:
# Create a build-like directory structure inside temp directory
build_dir = self._dir / "my_build_dir"
build_dir.mkdir(parents=True, exist_ok=True)
output_file = (
build_dir / "regenerator_outputs" / "remote_services.bazelrc"
)
output_file.parent.mkdir(parents=True, exist_ok=True)
template_file = (
self.source_dir / remote_services_utils.RBE_TEMPLATE_FILE
)
template_file.parent.mkdir(parents=True, exist_ok=True)
template_file.write_text(
r"""# AUTO-GENERATED
my_results_url={results_url}
"""
)
fake_settings = rbe_settings.fake(
bazel_enable=False,
cxx_enable=False,
link_enable=False,
rust_enable=False,
needs_reproxy=False,
needs_auth=False,
)
input_files = remote_services_utils.generate_remote_services_bazelrc(
fuchsia_dir=self.source_dir,
build_dir=build_dir,
output_path=output_file,
download_outputs="find-me",
environ={},
settings=fake_settings,
)
# Since RBE and ResultStore are disabled, the active inputs of RBE and RS profiles should NOT be in input_files!
rbe_paths = rbe_profile.resolve_paths(self.source_dir, {})
rs_paths = rs_profile.resolve_paths(self.source_dir, {})
for path in rbe_paths.active_inputs:
self.assertNotIn(path, input_files)
for path in rs_paths.active_inputs:
self.assertNotIn(path, input_files)
def test_generate_remote_services_bazelrc_loads_settings_from_build_dir(
self,
) -> None:
build_dir = self._dir / "my_build_dir2"
build_dir.mkdir(parents=True, exist_ok=True)
(build_dir / "rbe_settings.json").write_text(
"""{
"final": {
"bazel_enable": true,
"bazel_exec_strategy": "remote",
"bazel_download_outputs": "all",
"cxx_download_objects": true,
"cxx_enable": true,
"cxx_exec_strategy": "remote",
"cxx_minimalist_wrapper": true,
"link_download_unstripped_outputs": true,
"link_enable": true,
"link_exec_strategy": "remote",
"rust_download_rlibs": true,
"rust_download_unstripped_binaries": true,
"rust_enable": true,
"rust_exec_strategy": "remote",
"needs_reproxy": true,
"needs_auth": true
}
}"""
)
output_file = (
build_dir / "regenerator_outputs" / "remote_services.bazelrc"
)
output_file.parent.mkdir(parents=True, exist_ok=True)
template_file = (
self.source_dir / remote_services_utils.RBE_TEMPLATE_FILE
)
template_file.parent.mkdir(parents=True, exist_ok=True)
template_file.write_text("my_results_url={results_url}\n")
# Call with settings=None to exercise rbe_settings.load(build_dir)
input_files = remote_services_utils.generate_remote_services_bazelrc(
fuchsia_dir=self.source_dir,
build_dir=build_dir,
output_path=output_file,
download_outputs="all",
environ={},
)
self.assertTrue(output_file.is_file())
if __name__ == "__main__":
unittest.main()