| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building Rust toolchain.""" |
| |
| import re |
| import contextlib |
| |
| from recipe_engine.config import Enum |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/cas_util", |
| "fuchsia/cipd_util", |
| "fuchsia/git_checkout", |
| "fuchsia/goma", |
| "fuchsia/macos_sdk", |
| "fuchsia/python3", |
| "fuchsia/status_check", |
| "fuchsia/toolchain", |
| "fuchsia/windows_sdk", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/platform", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PACKAGES = ["rust"] |
| |
| PROPERTIES = { |
| "repository": Property( |
| kind=str, |
| help="Git repository URL", |
| default="https://fuchsia.googlesource.com/third_party/rust", |
| ), |
| "revision": Property(kind=str, help="Revision", default=None), |
| "package": Property( |
| kind=Enum(*PACKAGES), help="Which package to build", default="rust" |
| ), |
| "builders": Property( |
| kind=dict, |
| help=( |
| "Mapping from platform name to list of builders to trigger " |
| "(Format: '<project>/<trigger name>')" |
| ), |
| ), |
| "test_suites": Property( |
| kind=list, |
| help="The Rust test suites to run", |
| default=["src/test/ui"], |
| ), |
| "run_on_target": Property( |
| kind=bool, |
| help="Whether to run tests on an emulated Fuchsia instance", |
| default=False, |
| ), |
| "local": Property( |
| kind=bool, |
| help="Whether the recipe is running in a local context", |
| default=False, |
| ), |
| "upload_to_cipd": Property( |
| kind=bool, |
| help="Whether to upload the build results to CIPD", |
| default=True, |
| ), |
| } |
| |
| |
| def RunSteps( |
| api, |
| repository, |
| revision, |
| package, |
| builders, |
| test_suites, |
| run_on_target, |
| local, |
| upload_to_cipd, |
| ): |
| use_goma = ( |
| not api.platform.arch == "arm" |
| and api.platform.bits == 64 |
| and not api.platform.is_win |
| and not local |
| ) |
| if use_goma: |
| api.goma.ensure() |
| goma_context = api.goma.build_with_goma() |
| else: |
| goma_context = contextlib.nullcontext() |
| |
| host_platform = api.cipd_util.platform_name |
| |
| use_breakpad = host_platform == "linux-amd64" |
| |
| with api.step.nest("ensure_packages"): |
| with api.context(infra_steps=True): |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package("fuchsia/third_party/cmake/${platform}", "integration") |
| pkgs.add_package("fuchsia/third_party/ninja/${platform}", "integration") |
| pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration") |
| |
| # TODO(https://fxbug.dev/108193) switch back to using latest once unblocked |
| sysroot_tag = "git_revision:db18eec0b4f14b6b16174aa2b91e016663157376" |
| pkgs.add_package("fuchsia/third_party/sysroot/linux", sysroot_tag, "linux") |
| |
| # We don't have SDK for linux-arm64 or win, but we only need sysroot. |
| if ( |
| api.platform.arch == "arm" and api.platform.bits == 64 |
| ) or api.platform.is_win: |
| pkgs.add_package("fuchsia/sdk/core/linux-amd64", "latest", "sdk") |
| else: |
| pkgs.add_package("fuchsia/sdk/core/${platform}", "latest", "sdk") |
| if use_breakpad: |
| pkgs.add_package( |
| "fuchsia/tools/breakpad/${platform}", "latest", "breakpad" |
| ) |
| cipd_dir = api.path["start_dir"].join("cipd") |
| api.cipd.ensure(cipd_dir, pkgs) |
| sdk_dir = cipd_dir.join("sdk") |
| |
| rust_dir, revision = api.git_checkout( |
| repository, fallback_ref=revision, recursive=True |
| ) |
| |
| # build rust |
| staging_dir = api.path["start_dir"].join("staging") |
| build_dir = staging_dir.join("build") |
| api.file.ensure_directory("build", build_dir) |
| pkg_dir = staging_dir.join("rust") |
| api.file.ensure_directory("create pkg_dir", pkg_dir) |
| |
| with api.macos_sdk(), api.windows_sdk(), goma_context: |
| if api.platform.name == "linux": |
| host_sysroot = cipd_dir.join("linux") |
| elif api.platform.name == "mac": |
| # TODO(fxbug.dev/3043): Eventually use our own hermetic sysroot as for Linux. |
| step_result = api.step( |
| "xcrun", |
| ["xcrun", "--sdk", "macosx", "--show-sdk-path"], |
| stdout=api.raw_io.output_text(name="sdk-path", add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "/some/xcode/path" |
| ), |
| ) |
| host_sysroot = step_result.stdout.strip() |
| elif api.platform.is_win: |
| host_sysroot = api.windows_sdk.sdk_dir |
| else: # pragma: no cover |
| assert False, "unsupported platform" |
| |
| # We use generate_config's default target list, which is all the targets |
| # needed for a Fuchsia build on the current host platform. We don't |
| # actually upload all of them, so some are just there to make sure they |
| # keep building (important for toolchain dev workflows and especially |
| # for Fuchsia targets). But each target std only adds about 90s-120s to |
| # the build, depending on the machine, so it's not that expensive. |
| config_file = "config.toml" |
| api.python3( |
| "generate config.toml", |
| [ |
| api.resource("generate_config.py"), |
| "config_toml", |
| "--clang-prefix", |
| cipd_dir, |
| "--prefix", |
| pkg_dir, |
| "--host-sysroot", |
| host_sysroot, |
| ] |
| + (["--goma-dir", api.goma.goma_dir] if use_goma else []), |
| stdout=api.raw_io.output_text( |
| leak_to=build_dir.join(config_file), add_output_log=True |
| ), |
| ) |
| |
| env_data = api.python3( |
| "generate environment", |
| [ |
| api.resource("generate_config.py"), |
| "environment", |
| "--source", |
| rust_dir, |
| "--clang-prefix", |
| cipd_dir, |
| "--sdk-dir", |
| sdk_dir, |
| "--revision", |
| revision, |
| "--linux-amd64-sysroot", |
| cipd_dir.join("linux"), |
| "--linux-arm64-sysroot", |
| cipd_dir.join("linux"), |
| ] |
| + (["--darwin-sysroot", host_sysroot] if api.platform.is_mac else []) |
| + (["--win-sysroot", host_sysroot] if api.platform.is_win else []), |
| stdout=api.raw_io.output_text(name="environment", add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "\n".join( |
| [ |
| "CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS=-Lnative=sdk/arch/arm64/sysroot/lib -Cpanic=abort", |
| "CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS=-Lnative=sdk/arch/x64/sysroot/lib -Cpanic=abort", |
| ] |
| ) |
| ), |
| ) |
| env = { |
| "TEST_TOOLCHAIN_TMP_DIR": api.path.mkdtemp(), |
| } |
| for line in env_data.stdout.splitlines(): |
| key, value = line.split("=", 1) |
| env[key] = value |
| |
| env_prefixes = {"PATH": [cipd_dir, cipd_dir.join("bin")]} |
| with api.context(cwd=build_dir, env=env, env_prefixes=env_prefixes): |
| api.python3( |
| "rust install", |
| [ |
| rust_dir.join("x.py"), |
| "install", |
| "--config", |
| config_file, |
| ], |
| ) |
| |
| run_tests( |
| api, |
| env, |
| rust_dir, |
| pkg_dir, |
| sdk_dir, |
| config_file, |
| test_suites, |
| run_on_target, |
| ) |
| |
| # We don't build the fuchsia runtime on windows |
| if not api.platform.is_win: |
| runtime_result = api.python3( |
| "generate runtime spec", |
| [api.resource("generate_config.py"), "runtime"], |
| stdout=api.json.output(), |
| ) |
| api.toolchain.strip_runtimes( |
| "generate runtime.json", |
| spec=runtime_result.stdout, |
| path=pkg_dir.join("lib"), |
| build_id_subpath="debug/.build-id", |
| readelf=cipd_dir.join("bin", "llvm-readelf"), |
| objcopy=cipd_dir.join("bin", "llvm-objcopy"), |
| dump_syms=( |
| cipd_dir.join("breakpad", "dump_syms", "dump_syms") |
| if use_breakpad |
| else None |
| ), |
| ) |
| |
| host_triple = api.toolchain.PLATFORM_TO_TRIPLE[host_platform] |
| api.file.copy( |
| "distribute llvm-profdata", |
| source=build_dir.join( |
| "fuchsia-build", host_triple, "llvm", "bin", "llvm-profdata" |
| ), |
| dest=pkg_dir.join("bin"), |
| ) |
| |
| if not local: |
| digest = api.cas_util.upload(pkg_dir, output_property="isolated") |
| |
| # package rust |
| step_result = api.step( |
| "rust version", |
| [pkg_dir.join("bin", "rustc"), "--version"], |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "rustc 1.19.0-nightly (75b056812 2017-05-15)" |
| ), |
| ) |
| m = re.search(r"rustc ([0-9a-z.-]+)", step_result.stdout) |
| assert m, "Cannot determine Rust version" |
| version = m.group(1) |
| |
| if not local and upload_to_cipd: |
| upload_targets( |
| api, |
| pkg_dir, |
| repository, |
| revision, |
| version, |
| ) |
| upload_host( |
| api, |
| pkg_dir, |
| repository, |
| revision, |
| version, |
| ) |
| |
| if host_platform in builders: |
| # Do a full integration build. This will use the just-built toolchain |
| # to build all of Fuchsia to check whether there are any regressions. |
| api.toolchain.trigger_build( |
| "rust_toolchain", |
| repository, |
| revision, |
| digest, |
| builders=builders[host_platform], |
| ) |
| |
| |
| def upload_host(api, pkg_dir, repository, revision, version): |
| """ |
| Depending on host triple, uploads host toolchain |
| """ |
| host_platform = api.cipd_util.platform_name |
| |
| if api.platform.is_win: |
| return |
| |
| host_manifests = [ |
| "manifest-cargo", |
| "manifest-clippy-preview", |
| "manifest-rust-analyzer-preview", |
| "manifest-rustc", |
| "manifest-rust-demangler-preview", |
| "manifest-rustfmt-preview", |
| "manifest-rust-src", |
| ] |
| version_name = "rust" |
| step_name = "generate host manifest file paths for {}".format(host_platform) |
| pkg_name = "fuchsia/third_party/rust/host/" + host_platform |
| |
| upload_package_from_toolchain_manifests( |
| api, |
| host_manifests, |
| pkg_dir, |
| repository, |
| revision, |
| version, |
| step_name, |
| pkg_name, |
| version_name, |
| ) |
| |
| |
| def upload_targets(api, pkg_dir, repository, revision, version): |
| """ |
| Depending on host platform, uploads necessary std library files |
| to build against target |
| """ |
| host_platform = api.cipd_util.platform_name |
| target_triple_groups = [] |
| target_triple_groups.extend( |
| { |
| "linux-amd64": [ |
| "aarch64-fuchsia,x86_64-fuchsia", |
| "aarch64-unknown-linux-gnu", |
| "x86_64-unknown-linux-gnu", |
| "wasm32-unknown-unknown", |
| ], |
| "linux-arm64": [], |
| "mac-amd64": [ |
| "x86_64-apple-darwin", |
| ], |
| "windows-amd64": [ |
| "x86_64-pc-windows-msvc", |
| ], |
| }[host_platform] |
| ) |
| |
| for target_triple_group in target_triple_groups: |
| manifests = [ |
| "manifest-rust-std-{}".format(x) for x in target_triple_group.split(",") |
| ] |
| # Fuchsia targets are bundled in one... just call triple "fuchsia" |
| target_name = ( |
| target_triple_group if "fuchsia" not in target_triple_group else "fuchsia" |
| ) |
| |
| version_name = "target_{}".format(target_name) |
| step_name = "generate target manifest file paths for {}".format(target_name) |
| pkg_name = "fuchsia/third_party/rust/target/" + target_name |
| |
| upload_package_from_toolchain_manifests( |
| api, |
| manifests, |
| pkg_dir, |
| repository, |
| revision, |
| version, |
| step_name, |
| pkg_name, |
| version_name, |
| ) |
| |
| |
| def upload_package_from_toolchain_manifests( |
| api, |
| manifests, |
| pkg_dir, |
| repository, |
| revision, |
| version, |
| step_name, |
| pkg_name, |
| version_name, |
| ): |
| manifest_file_step = api.python3( |
| step_name, |
| [ |
| api.resource("generate_manifest_file_paths.py"), |
| "read_manifests", |
| "--manifests", |
| ",".join(manifests), |
| "--pkg-dir", |
| pkg_dir, |
| ], |
| stdout=api.raw_io.output_text(name="package-manifest", add_output_log=True), |
| ) |
| |
| pkg_file_paths = [] |
| for line in manifest_file_step.stdout.splitlines(): |
| absolute_file_path = pkg_dir.join(line.strip()) |
| pkg_file_paths.append(absolute_file_path) |
| |
| is_host = pkg_name.startswith("fuchsia/third_party/rust/host") |
| is_fuchsia_target = pkg_name == "fuchsia/third_party/rust/target/fuchsia" |
| if is_host: |
| # Only add "bin/llvm-profdata" for host toolings. |
| llvm_profdata_file_path = pkg_dir.join("bin", "llvm-profdata") |
| pkg_file_paths.append(llvm_profdata_file_path) |
| if is_fuchsia_target: |
| # "lib/debug/.build-id" and "lib/runtime.json" needed for Fuchsia targets. |
| build_id_file_path = pkg_dir.join("lib", "debug", ".build-id") |
| runtime_json = pkg_dir.join("lib", "runtime.json") |
| pkg_file_paths.append(build_id_file_path) |
| pkg_file_paths.append(runtime_json) |
| |
| upload_package( |
| api, |
| pkg_name, |
| pkg_dir, |
| pkg_paths=pkg_file_paths, |
| revision=revision, |
| repository=repository, |
| version=version, |
| version_name=version_name, |
| ) |
| |
| |
| def upload_package( |
| api, pkg_name, pkg_root, revision, repository, version, version_name=None, **kwargs |
| ): |
| git_ref = api.buildbucket.build.input.gitiles_commit.ref |
| cipd_refs = () |
| if git_ref == "refs/heads/main": |
| cipd_refs = ("latest",) |
| elif git_ref == "refs/heads/upstream/beta": |
| cipd_refs = ("latest_beta",) |
| api.cipd_util.upload_package( |
| pkg_name, |
| pkg_root, |
| search_tag={"git_revision": revision}, |
| repository=repository, |
| metadata=[("version", version)] + ([("ref", git_ref)] if git_ref else []), |
| refs=cipd_refs, |
| name=version_name, |
| **kwargs, |
| ) |
| |
| |
| def run_tests( |
| api, env, rust_dir, pkg_dir, sdk_dir, config_file, test_suites, run_on_target |
| ): |
| # Run host tests. These are just the compile-only "ui" tests for |
| # fuchsia targets. |
| |
| test_toolchain = api.resource("test_toolchain.py") |
| |
| for target, target_arch, host_arch in [ |
| ("aarch64-fuchsia", "arm64", "arm"), |
| ("x86_64-fuchsia", "x64", "intel"), |
| ]: |
| # Running the full test suite requires an emulator, which runs too slowly unless hardware |
| # acceleration is available. This requires the target arch and host arch to match. |
| run_target_test_suite = run_on_target and api.platform.arch == host_arch |
| |
| if run_target_test_suite: |
| # Start test environment |
| api.python3( |
| "start test environment", |
| [ |
| test_toolchain, |
| "start", |
| "--rust", |
| pkg_dir, |
| "--sdk", |
| sdk_dir, |
| "--target-arch", |
| target_arch, |
| "--verbose", |
| ], |
| ) |
| |
| # Getting target rustc flags... |
| target_env = "CARGO_TARGET_{}_RUSTFLAGS".format( |
| target.upper().replace("-", "_") |
| ) |
| env_target_flags = env[target_env].split(" ") |
| |
| target_flags = ["-Zpanic-abort-tests"] |
| for env_target_flag in env_target_flags: |
| if env_target_flag.startswith("-Lnative=") or env_target_flag.startswith( |
| "-L=" |
| ): |
| # `-Lnative=` does not work for compiletest... nor does `-L=` |
| l_prefix = ( |
| len("-Lnative=") |
| if env_target_flag.startswith("-Lnative=") |
| else len("-L=") |
| ) |
| target_flags += ["-L", env_target_flag[l_prefix:]] |
| else: |
| target_flags += [env_target_flag] |
| |
| # Getting host rustc flags... |
| ARCH_TO_SDK_ARCH = { |
| "intel": "x64", |
| "arm": "arm64", |
| } |
| sdk_arch = ARCH_TO_SDK_ARCH[api.platform.arch] |
| sdk_arch_path = sdk_dir.join("arch", sdk_arch) |
| host_sysroot_lib_path = sdk_arch_path.join("sysroot", "lib") |
| host_sdk_lib_path = sdk_arch_path.join("lib") |
| host_flags = [ |
| "-L", |
| str(host_sysroot_lib_path), |
| "-L", |
| str(host_sdk_lib_path), |
| ] |
| |
| cmd = [ |
| rust_dir.join("x.py"), |
| "test", |
| " ".join(test_suites), |
| "--config", |
| config_file, |
| "--stage=2", |
| "--target={}".format(target), |
| ] |
| |
| extra_test_flags = [] |
| if run_target_test_suite: |
| cmd += [ |
| "--run=always", |
| "--jobs=1", |
| ] |
| |
| extra_test_flags += [ |
| "--remote-test-client", |
| str(api.resource("test_toolchain.py")), |
| ] |
| else: |
| cmd += ["--run=never"] |
| |
| test_args = [ |
| arg |
| for flag in host_flags |
| for arg in ["--test-args", "--host-rustcflags", "--test-args", flag] |
| ] |
| test_args += [ |
| arg |
| for flag in target_flags |
| for arg in ["--test-args", "--target-rustcflags", "--test-args", flag] |
| ] |
| test_args += [arg for flag in extra_test_flags for arg in ["--test-args", flag]] |
| |
| cmd += test_args |
| |
| try: |
| api.python3( |
| "rust test {}".format(target), |
| cmd, |
| ) |
| except api.step.StepFailure: |
| # Only fail entire build if `rust_test` builders fail |
| if run_target_test_suite: |
| raise |
| finally: |
| if run_target_test_suite: |
| api.python3( |
| "stop test environment", |
| [ |
| test_toolchain, |
| "stop", |
| ], |
| ) |
| |
| |
| def GenTests(api): |
| builders = { |
| "linux-amd64": ["fuchsia/linux-x64-builder"], |
| "mac-amd64": ["fuchsia/mac-x64-builder"], |
| } |
| |
| recipe_arch_to_cipd_arch = { |
| "intel": "amd64", |
| "arm": "arm64", |
| } |
| |
| for platform in ("linux", "mac", "win"): |
| for package in ("rust",): |
| for arch in ("intel", "arm") if platform == "linux" else ("intel",): |
| for branch in ("main", "upstream/beta"): |
| # pylint: disable=use-maxsplit-arg |
| test = ( |
| api.buildbucket_util.test( |
| "%s_%s_%s_%s" |
| % (package, arch, platform, branch.split("/")[-1]), |
| repo="third_party/rust", |
| git_ref="refs/heads/%s" % (branch), |
| ) |
| + api.platform.name(platform) |
| + api.platform.arch(arch) |
| + api.properties( |
| package=package, |
| builders=builders, |
| ) |
| ) |
| if platform != "win": |
| test += api.step_data( |
| "generate runtime spec", stdout=api.json.output([]) |
| ) |
| test += api.step_data( |
| "generate host manifest file paths for {}-{}".format( |
| platform, recipe_arch_to_cipd_arch[arch] |
| ), |
| stdout=api.raw_io.output_text("relative/file/path"), |
| ) |
| yield (test) |
| yield ( |
| api.buildbucket_util.test("rust_intel_linux_full", repo="third_party/rust") |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + api.properties( |
| package="rust", |
| builders=builders, |
| test_suites=["src/test/ui"], |
| run_on_target=True, |
| ) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| ) |
| # "compiling test" failures will still pass the build (see `run_on_target=False`) |
| yield ( |
| api.buildbucket_util.test("failing_compile_tests", repo="third_party/rust") |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + api.properties( |
| package="rust", |
| builders=builders, |
| test_suites=["src/test/ui"], |
| run_on_target=False, |
| ) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| + api.step_data("rust test x86_64-fuchsia", retcode=1) |
| ) |
| # "emulator test" failures will fail the whole build (see `run_on_target=True`) |
| yield ( |
| api.buildbucket_util.test( |
| "failing_emulator_tests", repo="third_party/rust", status="failure" |
| ) |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + api.properties( |
| package="rust", |
| builders=builders, |
| test_suites=["src/test/ui"], |
| run_on_target=True, |
| ) |
| + api.step_data("rust test x86_64-fuchsia", retcode=1) |
| ) |
| yield ( |
| api.status_check.test("manual") |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + api.properties( |
| package="rust", |
| builders=builders, |
| ) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| ) |
