| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Recipe for building Rust toolchain.""" |
| |
| import re |
| import contextlib |
| |
| from PB.recipes.fuchsia.contrib.rust_toolchain import InputProperties |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/cas_util", |
| "fuchsia/cipd_util", |
| "fuchsia/git_checkout", |
| "fuchsia/gitiles", |
| "fuchsia/goma", |
| "fuchsia/macos_sdk", |
| "fuchsia/python3", |
| "fuchsia/toml", |
| "fuchsia/toolchain", |
| "fuchsia/windows_sdk", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/platform", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| "recipe_engine/url", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps(api, props): |
| # Always use upstream beta for windows (http://fxbug.dev/121692) |
| use_upstream_beta = props.use_upstream_beta or api.platform.is_win |
| |
| use_goma = ( |
| not api.platform.arch == "arm" |
| and api.platform.bits == 64 |
| and not api.platform.is_win |
| and not props.local |
| ) |
| if use_goma: |
| goma_context = api.goma() |
| else: |
| goma_context = contextlib.nullcontext() |
| |
| rust_dir, revision = api.git_checkout( |
| props.repository, fallback_ref=props.default_ref, recursive=True |
| ) |
| |
| host_platform = api.cipd_util.platform_name |
| use_breakpad = host_platform == "linux-amd64" |
| cipd_dir = fetch_cipd_packages( |
| api, rust_dir, host_platform, use_breakpad, use_upstream_beta |
| ) |
| |
| with api.step.nest("update sdk"): |
| sdk_dir = api.path["start_dir"].join("sdk") |
| api.file.copytree("copy sdk", cipd_dir.join("sdk"), sdk_dir) |
| api.file.chmod( |
| "chmod pkg/sysroot/meta.json", |
| sdk_dir.join("pkg", "sysroot", "meta.json"), |
| "0644", |
| ) |
| # Generate artificial riscv64 sysroot since Fuchsia SDK doesn't contain one yet. |
| generate_sysroot = api.gitiles.fetch( |
| "fuchsia.googlesource.com", |
| "fuchsia", |
| "scripts/clang/generate_sysroot.py", |
| # This is base64 encoded `#!/usr/bin/env python3`. |
| default_test_data="IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMwo=", |
| ) |
| api.step( |
| "generate riscv64 sysroot", |
| cmd=[ |
| "vpython3", |
| api.raw_io.input(generate_sysroot, ".py"), |
| f"--sdk-dir={sdk_dir}", |
| "--arch=riscv64", |
| f"--ifs-path={cipd_dir.join('bin', 'llvm-ifs')}", |
| ], |
| ) |
| |
| # build rust |
| staging_dir = api.path["start_dir"].join("staging") |
| build_dir = staging_dir.join("build") |
| api.file.ensure_directory("build", build_dir) |
| pkg_dir = staging_dir.join("rust") |
| api.file.ensure_directory("create pkg_dir", pkg_dir) |
| |
| with api.macos_sdk(), api.windows_sdk(), goma_context: |
| if api.platform.name == "linux": |
| host_sysroot = cipd_dir.join("linux") |
| elif api.platform.name == "mac": |
| # TODO(fxbug.dev/3043): Eventually use our own hermetic sysroot as for Linux. |
| host_sysroot = api.macos_sdk.sysroot |
| elif api.platform.is_win: |
| host_sysroot = api.windows_sdk.sdk_dir |
| else: # pragma: no cover |
| assert False, "unsupported platform" |
| |
| # We use generate_config's default target list, which is all the targets |
| # needed for a Fuchsia build on the current host platform. We don't |
| # actually upload all of them, so some are just there to make sure they |
| # keep building (important for toolchain dev workflows and especially |
| # for Fuchsia targets). But each target std only adds about 90s-120s to |
| # the build, depending on the machine, so it's not that expensive. |
| config_file = "config.toml" |
| beta_dir = cipd_dir.join("beta") |
| api.python3( |
| "generate config.toml", |
| [ |
| api.resource("generate_config.py"), |
| "config_toml", |
| "--clang-prefix", |
| cipd_dir, |
| "--prefix", |
| pkg_dir, |
| "--host-sysroot", |
| host_sysroot, |
| ] |
| + ([] if use_upstream_beta else ["--beta", beta_dir]) |
| + (["--goma-dir", api.goma.goma_dir] if use_goma else []), |
| stdout=api.raw_io.output_text( |
| leak_to=build_dir.join(config_file), add_output_log=True |
| ), |
| ) |
| |
| env_data = api.python3( |
| "generate environment", |
| [ |
| api.resource("generate_config.py"), |
| "environment", |
| "--source", |
| rust_dir, |
| "--clang-prefix", |
| cipd_dir, |
| "--sdk-dir", |
| sdk_dir, |
| "--revision", |
| revision, |
| "--linux-amd64-sysroot", |
| cipd_dir.join("linux"), |
| "--linux-arm64-sysroot", |
| cipd_dir.join("linux"), |
| ] |
| + (["--darwin-sysroot", host_sysroot] if api.platform.is_mac else []) |
| + (["--win-sysroot", host_sysroot] if api.platform.is_win else []), |
| stdout=api.raw_io.output_text(name="environment", add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "\n".join( |
| [ |
| "CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS=-Lnative=sdk/arch/arm64/sysroot/lib -Cpanic=abort", |
| "CARGO_TARGET_RISCV64GC_UNKNOWN_FUCHSIA_RUSTFLAGS=-Lnative=sdk/arch/riscv64/sysroot/lib -Cpanic=abort", |
| "CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS=-Lnative=sdk/arch/x64/sysroot/lib -Cpanic=abort", |
| ] |
| ) |
| ), |
| ) |
| env = { |
| "TEST_TOOLCHAIN_TMP_DIR": api.path.mkdtemp(), |
| } |
| for line in env_data.stdout.splitlines(): |
| key, value = line.split("=", 1) |
| env[key] = value |
| |
| env_prefixes = {"PATH": [cipd_dir, cipd_dir.join("bin")]} |
| with api.context(cwd=build_dir, env=env, env_prefixes=env_prefixes): |
| api.python3( |
| "rust install", |
| [ |
| rust_dir.join("x.py"), |
| "install", |
| "--config", |
| config_file, |
| ], |
| ) |
| |
| run_tests( |
| api, |
| env, |
| rust_dir, |
| pkg_dir, |
| sdk_dir, |
| config_file, |
| props.test_suites, |
| props.run_on_target, |
| ) |
| |
| # We don't build the fuchsia runtime on windows |
| if not api.platform.is_win: |
| runtime_result = api.python3( |
| "generate runtime spec", |
| [api.resource("generate_config.py"), "runtime"], |
| stdout=api.json.output(), |
| ) |
| api.toolchain.strip_runtimes( |
| "generate runtime.json", |
| spec=runtime_result.stdout, |
| path=pkg_dir.join("lib"), |
| build_id_subpath="debug/.build-id", |
| readelf=cipd_dir.join("bin", "llvm-readelf"), |
| objcopy=cipd_dir.join("bin", "llvm-objcopy"), |
| dump_syms=( |
| cipd_dir.join("breakpad", "dump_syms", "dump_syms") |
| if use_breakpad |
| else None |
| ), |
| ) |
| |
| host_triple = api.toolchain.PLATFORM_TO_TRIPLE[host_platform] |
| api.file.copy( |
| "distribute llvm-profdata", |
| source=build_dir.join( |
| "fuchsia-build", host_triple, "llvm", "bin", "llvm-profdata" |
| ), |
| dest=pkg_dir.join("bin"), |
| ) |
| |
| if not props.local: |
| digest = api.cas_util.upload(pkg_dir, output_property="isolated") |
| |
| # package rust |
| step_result = api.step( |
| "rust version", |
| [pkg_dir.join("bin", "rustc"), "--version"], |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "rustc 1.19.0-nightly (75b056812 2017-05-15)" |
| ), |
| ) |
| m = re.search(r"rustc ([0-9a-z.-]+)", step_result.stdout) |
| assert m, "Cannot determine Rust version" |
| version = m.group(1) |
| |
| if not props.local and props.upload_to_cipd: |
| upload_targets( |
| api, |
| pkg_dir, |
| props.repository, |
| revision, |
| version, |
| ) |
| upload_host( |
| api, |
| pkg_dir, |
| props.repository, |
| revision, |
| version, |
| ) |
| |
| if host_platform in props.builders: |
| # Do a full integration build. This will use the just-built toolchain |
| # to build all of Fuchsia to check whether there are any regressions. |
| api.toolchain.trigger_build( |
| "rust_toolchain", |
| props.repository, |
| revision, |
| digest, |
| builders=props.builders[host_platform], |
| ) |
| |
| |
| def fetch_cipd_packages(api, rust_dir, host_platform, use_breakpad, use_upstream_beta): |
| with api.step.nest("ensure_packages"): |
| with api.context(infra_steps=True): |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration") |
| pkgs.add_package("fuchsia/third_party/cmake/${platform}", "integration") |
| pkgs.add_package("fuchsia/third_party/ninja/${platform}", "integration") |
| |
| if not api.platform.is_win: |
| pkgs.add_package("fuchsia/third_party/make/${platform}", "version:4.3") |
| |
| if use_upstream_beta: # pragma: no cover |
| # with upstream's beta, you need to use this old sysroot version |
| # because it contains libgcc_s (https://fxbug.dev/108193) |
| legacy_sysroot_tag = ( |
| "git_revision:db18eec0b4f14b6b16174aa2b91e016663157376" |
| ) |
| pkgs.add_package( |
| "fuchsia/third_party/sysroot/linux", legacy_sysroot_tag, "linux" |
| ) |
| else: |
| pkgs.add_package( |
| "fuchsia/third_party/sysroot/linux", "integration", "linux" |
| ) |
| |
| beta_date = api.file.read_json( |
| "read stage0 version info", |
| rust_dir.join("src").join("stage0.json"), |
| test_data={"compiler": {"date": "01-01-1970"}}, |
| )["compiler"]["date"] |
| |
| api.url.get_file( |
| f"https://static.rust-lang.org/dist/{beta_date}/channel-rust-beta.toml", |
| "beta.toml", |
| "download corresponding rust manifest", |
| ) |
| beta_rust_commit_hash = api.toml.read_file( |
| "parse commit_hash from manifest", |
| "beta.toml", |
| test_data={"pkg": {"rust": {"git_commit_hash": "abad1dea"}}}, |
| )["pkg"]["rust"]["git_commit_hash"] |
| |
| host_triple = api.toolchain.PLATFORM_TO_TRIPLE[host_platform] |
| pkgs.add_package( |
| f"fuchsia/third_party/rust/host/{host_platform}", |
| f"git_revision:{beta_rust_commit_hash}", |
| "beta", |
| ) |
| pkgs.add_package( |
| f"fuchsia/third_party/rust/target/{host_triple}", |
| f"git_revision:{beta_rust_commit_hash}", |
| "beta", |
| ) |
| |
| # We don't have SDK for linux-arm64 or win, but we only need sysroot. |
| if ( |
| api.platform.arch == "arm" and api.platform.bits == 64 |
| ) or api.platform.is_win: |
| pkgs.add_package("fuchsia/sdk/core/linux-amd64", "latest", "sdk") |
| else: |
| pkgs.add_package("fuchsia/sdk/core/${platform}", "latest", "sdk") |
| if use_breakpad: |
| pkgs.add_package( |
| "fuchsia/tools/breakpad/${platform}", "integration", "breakpad" |
| ) |
| cipd_dir = api.path["start_dir"].join("cipd") |
| api.cipd.ensure(cipd_dir, pkgs) |
| return cipd_dir |
| |
| |
| def upload_host(api, pkg_dir, repository, revision, version): |
| """ |
| Depending on host triple, uploads host toolchain |
| """ |
| host_platform = api.cipd_util.platform_name |
| |
| if api.platform.is_win: |
| return |
| |
| host_manifests = [ |
| "manifest-cargo", |
| "manifest-clippy-preview", |
| "manifest-rust-analyzer-preview", |
| "manifest-rustc", |
| "manifest-rust-demangler-preview", |
| "manifest-rustfmt-preview", |
| "manifest-rust-src", |
| ] |
| version_name = "rust" |
| step_name = f"generate host manifest file paths for {host_platform}" |
| pkg_name = "fuchsia/third_party/rust/host/" + host_platform |
| |
| upload_package_from_toolchain_manifests( |
| api, |
| host_manifests, |
| pkg_dir, |
| repository, |
| revision, |
| version, |
| step_name, |
| pkg_name, |
| version_name, |
| ) |
| |
| |
| def upload_targets(api, pkg_dir, repository, revision, version): |
| """ |
| Depending on host platform, uploads necessary std library files |
| to build against target |
| """ |
| host_platform = api.cipd_util.platform_name |
| target_triple_groups = [] |
| target_triple_groups.extend( |
| { |
| "linux-amd64": [ |
| "aarch64-fuchsia,riscv64gc-unknown-fuchsia,x86_64-fuchsia", |
| "aarch64-unknown-linux-gnu", |
| "x86_64-unknown-linux-gnu", |
| "wasm32-unknown-unknown", |
| "thumbv7m-none-eabi", |
| ], |
| "linux-arm64": [], |
| "mac-amd64": [ |
| "x86_64-apple-darwin", |
| ], |
| "windows-amd64": [ |
| "x86_64-pc-windows-msvc", |
| ], |
| }[host_platform] |
| ) |
| |
| for target_triple_group in target_triple_groups: |
| manifests = [f"manifest-rust-std-{x}" for x in target_triple_group.split(",")] |
| # Fuchsia targets are bundled in one... just call triple "fuchsia" |
| target_name = ( |
| target_triple_group if "fuchsia" not in target_triple_group else "fuchsia" |
| ) |
| |
| version_name = f"target_{target_name}" |
| step_name = f"generate target manifest file paths for {target_name}" |
| pkg_name = "fuchsia/third_party/rust/target/" + target_name |
| |
| upload_package_from_toolchain_manifests( |
| api, |
| manifests, |
| pkg_dir, |
| repository, |
| revision, |
| version, |
| step_name, |
| pkg_name, |
| version_name, |
| ) |
| |
| |
| def upload_package_from_toolchain_manifests( |
| api, |
| manifests, |
| pkg_dir, |
| repository, |
| revision, |
| version, |
| step_name, |
| pkg_name, |
| version_name, |
| ): |
| manifest_file_step = api.python3( |
| step_name, |
| [ |
| api.resource("generate_manifest_file_paths.py"), |
| "read_manifests", |
| "--manifests", |
| ",".join(manifests), |
| "--pkg-dir", |
| pkg_dir, |
| ], |
| stdout=api.raw_io.output_text(name="package-manifest", add_output_log=True), |
| ) |
| |
| pkg_file_paths = [] |
| for line in manifest_file_step.stdout.splitlines(): |
| absolute_file_path = pkg_dir.join(line.strip()) |
| pkg_file_paths.append(absolute_file_path) |
| |
| is_host = pkg_name.startswith("fuchsia/third_party/rust/host") |
| is_fuchsia_target = pkg_name == "fuchsia/third_party/rust/target/fuchsia" |
| if is_host: |
| # Only add "bin/llvm-profdata" for host toolings. |
| llvm_profdata_file_path = pkg_dir.join("bin", "llvm-profdata") |
| pkg_file_paths.append(llvm_profdata_file_path) |
| if is_fuchsia_target: |
| # "lib/debug/.build-id" and "lib/runtime.json" needed for Fuchsia targets. |
| build_id_file_path = pkg_dir.join("lib", "debug", ".build-id") |
| runtime_json = pkg_dir.join("lib", "runtime.json") |
| pkg_file_paths.append(build_id_file_path) |
| pkg_file_paths.append(runtime_json) |
| |
| upload_package( |
| api, |
| pkg_name, |
| pkg_dir, |
| pkg_paths=pkg_file_paths, |
| revision=revision, |
| repository=repository, |
| version=version, |
| version_name=version_name, |
| ) |
| |
| |
| def upload_package( |
| api, pkg_name, pkg_root, revision, repository, version, version_name=None, **kwargs |
| ): |
| git_ref = api.buildbucket.build.input.gitiles_commit.ref |
| cipd_refs = () |
| if git_ref == "refs/heads/main": |
| cipd_refs = ("latest",) |
| elif git_ref == "refs/heads/upstream/beta": |
| cipd_refs = ("latest_beta",) |
| api.cipd_util.upload_package( |
| pkg_name, |
| pkg_root, |
| search_tag={"git_revision": revision}, |
| repository=repository, |
| metadata=[("version", version)] + ([("ref", git_ref)] if git_ref else []), |
| refs=cipd_refs, |
| name=version_name, |
| **kwargs, |
| ) |
| |
| |
| def run_tests( |
| api, env, rust_dir, pkg_dir, sdk_dir, config_file, test_suites, run_on_target |
| ): |
| # Run host tests. These are just the compile-only "ui" tests for |
| # fuchsia targets. |
| |
| test_toolchain_script = rust_dir.join( |
| "src", "ci", "docker", "scripts", "fuchsia-test-runner.py" |
| ) |
| |
| for target, host_arch in [ |
| ("aarch64-fuchsia", "arm"), |
| ("x86_64-fuchsia", "intel"), |
| ]: |
| # Running the full test suite requires an emulator, which runs too slowly unless hardware |
| # acceleration is available. This requires the target arch and host arch to match. |
| run_target_test_suite = run_on_target and api.platform.arch == host_arch |
| |
| if run_target_test_suite: |
| # Start test environment |
| api.python3( |
| "start test environment", |
| [ |
| test_toolchain_script, |
| "start", |
| "--rust", |
| pkg_dir, |
| "--sdk", |
| sdk_dir, |
| "--target", |
| target, |
| "--verbose", |
| ], |
| ) |
| |
| # Getting target rustc flags... |
| target_env = f"CARGO_TARGET_{target.upper().replace('-', '_')}_RUSTFLAGS" |
| env_target_flags = env[target_env].split(" ") |
| |
| target_flags = ["-Zpanic-abort-tests"] |
| for env_target_flag in env_target_flags: |
| if env_target_flag.startswith("-Lnative=") or env_target_flag.startswith( |
| "-L=" |
| ): |
| # `-Lnative=` does not work for compiletest... nor does `-L=` |
| l_prefix = ( |
| len("-Lnative=") |
| if env_target_flag.startswith("-Lnative=") |
| else len("-L=") |
| ) |
| target_flags += ["-L", env_target_flag[l_prefix:]] |
| else: |
| target_flags += [env_target_flag] |
| |
| # Getting host rustc flags... |
| ARCH_TO_SDK_ARCH = { |
| "intel": "x64", |
| "arm": "arm64", |
| } |
| sdk_arch = ARCH_TO_SDK_ARCH[api.platform.arch] |
| sdk_arch_path = sdk_dir.join("arch", sdk_arch) |
| host_sysroot_lib_path = sdk_arch_path.join("sysroot", "lib") |
| host_sdk_lib_path = sdk_arch_path.join("lib") |
| host_flags = [ |
| "-L", |
| str(host_sysroot_lib_path), |
| "-L", |
| str(host_sdk_lib_path), |
| ] |
| |
| cmd = [ |
| rust_dir.join("x.py"), |
| "test", |
| " ".join(test_suites), |
| "--config", |
| config_file, |
| "--stage=2", |
| f"--target={target}", |
| ] |
| |
| extra_test_flags = [] |
| if run_target_test_suite: |
| cmd += [ |
| "--run=always", |
| "--jobs=1", |
| ] |
| |
| extra_test_flags += [ |
| "--remote-test-client", |
| str(test_toolchain_script), |
| ] |
| else: |
| cmd += ["--run=never"] |
| |
| test_args = [ |
| arg |
| for flag in host_flags |
| for arg in ["--test-args", "--host-rustcflags", "--test-args", flag] |
| ] |
| test_args += [ |
| arg |
| for flag in target_flags |
| for arg in ["--test-args", "--target-rustcflags", "--test-args", flag] |
| ] |
| test_args += [arg for flag in extra_test_flags for arg in ["--test-args", flag]] |
| |
| cmd += test_args |
| |
| try: |
| api.python3(f"rust test {target}", cmd) |
| except api.step.StepFailure: |
| # Only fail entire build if `rust_test` builders fail |
| if run_target_test_suite: |
| raise |
| finally: |
| if run_target_test_suite: |
| api.python3( |
| "stop test environment", |
| [ |
| test_toolchain_script, |
| "stop", |
| ], |
| ) |
| |
| |
| def GenTests(api): |
| recipe_arch_to_cipd_arch = { |
| "intel": "amd64", |
| "arm": "arm64", |
| } |
| |
| def properties(**kwargs): |
| default_props = { |
| "repository": "https://fuchsia.googlesource.com/third_party/rust", |
| "builders": { |
| "linux-amd64": ["fuchsia/linux-x64-builder"], |
| "mac-amd64": ["fuchsia/mac-x64-builder"], |
| }, |
| "upload_to_cipd": True, |
| "use_upstream_beta": True, |
| "test_suites": ["tests/ui"], |
| } |
| return api.properties(**{**default_props, **kwargs}) |
| |
| for platform in ("linux", "mac", "win"): |
| for package in ("rust",): |
| for arch in ("intel", "arm") if platform == "linux" else ("intel",): |
| for branch in ("main", "upstream/beta"): |
| # pylint: disable=use-maxsplit-arg |
| test = ( |
| api.buildbucket_util.test( |
| f"{package}_{arch}_{platform}_{branch.split('/')[-1]}", |
| repo="third_party/rust", |
| git_ref=f"refs/heads/{branch}", |
| ) |
| + api.platform.name(platform) |
| + api.platform.arch(arch) |
| + properties(use_upstream_beta=False) |
| ) |
| if platform != "win": |
| test += api.step_data( |
| "generate runtime spec", stdout=api.json.output([]) |
| ) |
| test += api.step_data( |
| f"generate host manifest file paths for {platform}-{recipe_arch_to_cipd_arch[arch]}", |
| stdout=api.raw_io.output_text("relative/file/path"), |
| ) |
| yield test |
| yield ( |
| api.buildbucket_util.test("rust_intel_linux_full", repo="third_party/rust") |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + properties(run_on_target=True) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| ) |
| # "compiling test" failures will still pass the build (see `run_on_target=False`) |
| yield ( |
| api.buildbucket_util.test("failing_compile_tests", repo="third_party/rust") |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + properties() |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| + api.step_data("rust test x86_64-fuchsia", retcode=1) |
| ) |
| # "emulator test" failures will fail the whole build (see `run_on_target=True`) |
| yield ( |
| api.buildbucket_util.test( |
| "failing_emulator_tests", repo="third_party/rust", status="FAILURE" |
| ) |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + properties(run_on_target=True) |
| + api.step_data("rust test x86_64-fuchsia", retcode=1) |
| ) |
| yield ( |
| api.test("manual") |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + properties() |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| ) |
