| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building Rust toolchain.""" |
| |
| import re |
| import contextlib |
| |
| from recipe_engine.config import Enum |
| from recipe_engine.recipe_api import Property |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/cas_util", |
| "fuchsia/cipd_util", |
| "fuchsia/git", |
| "fuchsia/goma", |
| "fuchsia/macos_sdk", |
| "fuchsia/python3", |
| "fuchsia/status_check", |
| "fuchsia/toolchain", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/platform", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PACKAGES = ["rust"] |
| |
| PROPERTIES = { |
| "repository": Property( |
| kind=str, |
| help="Git repository URL", |
| default="https://fuchsia.googlesource.com/third_party/rust", |
| ), |
| "branch": Property(kind=str, help="Git branch", default="refs/heads/main"), |
| "revision": Property(kind=str, help="Revision", default=None), |
| "package": Property( |
| kind=Enum(*PACKAGES), help="Which package to build", default="rust" |
| ), |
| "builders": Property( |
| kind=dict, help="Mapping from platform name to list of builders to trigger" |
| ), |
| "full_test_suite": Property( |
| kind=bool, |
| help="Whether to run the Rust test suite on a Fuchsia emulator instance", |
| default=False, |
| ), |
| "local": Property( |
| kind=bool, |
| help="Whether the recipe is running in a local context", |
| default=False, |
| ), |
| } |
| |
| |
| def RunSteps( |
| api, repository, branch, revision, package, builders, full_test_suite, local |
| ): |
| use_goma = ( |
| not api.platform.arch == "arm" |
| and api.platform.bits == 64 |
| and not api.platform.is_win |
| and not local |
| ) |
| if use_goma: |
| api.goma.ensure() |
| goma_context = api.goma.build_with_goma() |
| else: |
| goma_context = contextlib.nullcontext() |
| |
| if not revision: |
| revision = api.git.get_remote_branch_head(repository, branch) |
| |
| host_platform = api.cipd_util.platform_name |
| |
| use_breakpad = host_platform == "linux-amd64" |
| |
| with api.step.nest("ensure_packages"): |
| with api.context(infra_steps=True): |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package("infra/3pp/tools/cmake/${platform}", "version:3.13.5") |
| pkgs.add_package("infra/3pp/tools/ninja/${platform}", "version:1.9.0") |
| pkgs.add_package("infra/3pp/tools/swig/${platform}", "version:4.0.2") |
| pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration") |
| pkgs.add_package("fuchsia/sysroot/linux-amd64", "latest", "linux-amd64") |
| pkgs.add_package("fuchsia/sysroot/linux-arm64", "latest", "linux-arm64") |
| # We don't have SDK for linux-arm64 or win, but we only need sysroot. |
| if ( |
| api.platform.arch == "arm" and api.platform.bits == 64 |
| ) or api.platform.is_win: |
| pkgs.add_package("fuchsia/sdk/core/linux-amd64", "latest", "sdk") |
| else: |
| pkgs.add_package("fuchsia/sdk/core/${platform}", "latest", "sdk") |
| if use_breakpad: |
| pkgs.add_package( |
| "fuchsia/tools/breakpad/${platform}", "latest", "breakpad" |
| ) |
| cipd_dir = api.path["start_dir"].join("cipd") |
| api.cipd.ensure(cipd_dir, pkgs) |
| sdk_dir = cipd_dir.join("sdk") |
| |
| with api.context(infra_steps=True): |
| rust_dir, _ = api.git.checkout(repository, ref=revision, recursive=True) |
| |
| # build rust |
| staging_dir = api.path["start_dir"].join("staging") |
| build_dir = staging_dir.join("build") |
| api.file.ensure_directory("build", build_dir) |
| pkg_dir = staging_dir.join("rust") |
| api.file.ensure_directory("create pkg_dir", pkg_dir) |
| |
| with api.macos_sdk(), goma_context: |
| if api.platform.name == "linux": |
| host_sysroot = cipd_dir.join(host_platform) |
| elif api.platform.name == "mac": |
| # TODO(fxbug.dev/3043): Eventually use our own hermetic sysroot as for Linux. |
| step_result = api.step( |
| "xcrun", |
| ["xcrun", "--sdk", "macosx", "--show-sdk-path"], |
| stdout=api.raw_io.output_text(name="sdk-path", add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "/some/xcode/path" |
| ), |
| ) |
| host_sysroot = step_result.stdout.strip() |
| else: # pragma: no cover |
| assert False, "unsupported platform" |
| |
| targets = [ |
| "x86_64-unknown-linux-gnu", |
| "aarch64-unknown-linux-gnu", |
| "wasm32-unknown-unknown", |
| ] |
| if api.platform.is_mac: |
| targets.append("x86_64-apple-darwin") |
| |
| config_file = "config.toml" |
| api.step( |
| "generate config.toml", |
| cmd=[ |
| "python", |
| "-u", |
| api.resource("generate_config.py"), |
| "config_toml", |
| "--targets", |
| ",".join(targets), |
| "--clang-prefix", |
| cipd_dir, |
| "--prefix", |
| pkg_dir, |
| "--host-sysroot", |
| host_sysroot, |
| ] |
| + (["--goma-dir", api.goma.goma_dir] if use_goma else []), |
| stdout=api.raw_io.output_text( |
| leak_to=build_dir.join(config_file), add_output_log=True |
| ), |
| ) |
| |
| env_data = api.step( |
| "generate environment", |
| cmd=[ |
| "python", |
| "-u", |
| api.resource("generate_config.py"), |
| "environment", |
| "--targets", |
| ",".join(targets), |
| "--source", |
| rust_dir, |
| "--clang-prefix", |
| cipd_dir, |
| "--sdk-dir", |
| sdk_dir, |
| "--revision", |
| revision, |
| # TODO(tmandry): Switch to multiarch so these don't need to be separate |
| "--linux-amd64-sysroot", |
| cipd_dir.join("linux-amd64"), |
| "--linux-arm64-sysroot", |
| cipd_dir.join("linux-arm64"), |
| ] |
| + (["--darwin-sysroot", host_sysroot] if api.platform.is_mac else []), |
| stdout=api.raw_io.output_text(name="environment", add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "\n".join( |
| [ |
| "CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS=-Lnative=sdk/arch/arm64/sysroot/lib -C panic=abort", |
| "CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS=-Lnative=sdk/arch/x64/sysroot/lib -C panic=abort", |
| ] |
| ) |
| ), |
| ) |
| env = { |
| "TEST_TOOLCHAIN_TMP_DIR": api.path.mkdtemp(), |
| } |
| for line in env_data.stdout.splitlines(): |
| key, value = line.split("=", 1) |
| env[key] = value |
| |
| env_prefixes = {"PATH": [cipd_dir, cipd_dir.join("bin")]} |
| with api.context(cwd=build_dir, env=env, env_prefixes=env_prefixes): |
| api.python3( |
| "rust install", |
| [ |
| rust_dir.join("x.py"), |
| "install", |
| "--config", |
| config_file, |
| ], |
| ) |
| |
| if package == "rust": |
| run_tests( |
| api, |
| env, |
| rust_dir, |
| pkg_dir, |
| sdk_dir, |
| config_file, |
| full_test_suite, |
| ) |
| |
| runtime_result = api.step( |
| "generate runtime spec", |
| cmd=["python", "-u", api.resource("generate_config.py"), "runtime"], |
| stdout=api.json.output(), |
| ) |
| api.toolchain.strip_runtimes( |
| "generate runtime.json", |
| spec=runtime_result.stdout, |
| path=pkg_dir.join("lib"), |
| build_id_subpath="debug/.build-id", |
| readelf=cipd_dir.join("bin", "llvm-readelf"), |
| objcopy=cipd_dir.join("bin", "llvm-objcopy"), |
| dump_syms=( |
| cipd_dir.join("breakpad", "dump_syms", "dump_syms") |
| if use_breakpad |
| else None |
| ), |
| ) |
| |
| host_triple = api.toolchain.PLATFORM_TO_TRIPLE[host_platform] |
| api.file.copy( |
| "distribute llvm-profdata", |
| source=build_dir.join( |
| "fuchsia-build", host_triple, "llvm", "bin", "llvm-profdata" |
| ), |
| dest=pkg_dir.join("bin"), |
| ) |
| |
| if not local: |
| digest = api.cas_util.upload(pkg_dir, output_property="isolated") |
| |
| # package rust |
| step_result = api.step( |
| "rust version", |
| [pkg_dir.join("bin", "rustc"), "--version"], |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "rustc 1.19.0-nightly (75b056812 2017-05-15)" |
| ), |
| ) |
| m = re.search(r"rustc ([0-9a-z.-]+)", step_result.stdout) |
| assert m, "Cannot determine Rust version" |
| version = m.group(1) |
| |
| if not api.buildbucket_util.is_tryjob and not local: |
| api.cipd_util.upload_package( |
| "fuchsia/third_party/%s/${platform}" % package, |
| pkg_dir, |
| search_tag={"git_revision": revision}, |
| repository=repository, |
| metadata=[("version", version)], |
| ) |
| |
| if package == "rust" and host_platform in builders: |
| # Do a full integration build. This will use the just-built toolchain |
| # to build all of Fuchsia to check whether there are any regressions. |
| api.toolchain.trigger_build( |
| "rust_toolchain", |
| repository, |
| revision, |
| digest, |
| builders=builders[host_platform], |
| ) |
| |
| |
| def run_tests(api, env, rust_dir, pkg_dir, sdk_dir, config_file, full_test_suite): |
| # Run host tests. These are just the compile-only "ui" tests for |
| # fuchsia targets. |
| |
| test_toolchain = api.resource("test_toolchain.py") |
| |
| for target, target_arch in [ |
| ("aarch64-fuchsia", "arm64"), |
| ("x86_64-fuchsia", "x64"), |
| ]: |
| # TODO(dkoloski): ungate target arch |
| if full_test_suite and target_arch == "x64": |
| # Start test environment |
| api.python3( |
| "start test environment", |
| [ |
| test_toolchain, |
| "start", |
| "--rust", |
| pkg_dir, |
| "--sdk", |
| sdk_dir, |
| "--target-arch", |
| target_arch, |
| "--verbose", |
| ], |
| ) |
| |
| target_env = "CARGO_TARGET_{}_RUSTFLAGS".format( |
| target.upper().replace("-", "_") |
| ) |
| target_flags = env[target_env].split(" ") |
| target_flags.append("-Zpanic-abort-tests") |
| test_args = [ |
| arg for flag in target_flags for arg in ["--target-rustcflags", flag] |
| ] |
| test_args.append("--target-panic=abort") |
| |
| cmd = [ |
| rust_dir.join("x.py"), |
| "test", |
| "--config", |
| config_file, |
| "--stage=2", |
| "--target={}".format(target), |
| ] |
| |
| # TODO(dkoloski): ungate target arch |
| if full_test_suite and target_arch == "x64": |
| ARCH_TO_SDK_ARCH = { |
| "intel": "x64", |
| "arm": "arm64", |
| } |
| sdk_arch = ARCH_TO_SDK_ARCH[api.platform.arch] |
| sdk_arch_path = sdk_dir.join("arch", sdk_arch) |
| sysroot_lib_path = sdk_arch_path.join("sysroot", "lib") |
| sdk_lib_path = sdk_arch_path.join("lib") |
| |
| cmd += [ |
| # TODO(dkoloski): run more of the rust test suite |
| "src/test/ui/abi/abi-sysv64-arg-passing.rs", |
| "--run=always", |
| "--jobs=1", |
| "--rustc-args", |
| "-C panic=abort -Zpanic_abort_tests -L {} -L {}".format( |
| sysroot_lib_path, sdk_lib_path |
| ), |
| ] |
| |
| test_args += [ |
| "--remote-test-client", |
| str(api.resource("test_toolchain.py")), |
| ] |
| else: |
| cmd += [ |
| "src/test/ui", |
| "--run=never", |
| ] |
| |
| cmd += [ |
| "--test-args", |
| " ".join(test_args), |
| ] |
| |
| try: |
| api.python3( |
| "rust test {}".format(target), |
| cmd, |
| ) |
| except api.step.StepFailure: |
| # Don't fail the entire build if tests fail, for now. |
| pass |
| finally: |
| # TODO(dkoloski): ungate target arch |
| if full_test_suite and target_arch == "x64": |
| api.python3( |
| "stop test environment", |
| [ |
| test_toolchain, |
| "stop", |
| ], |
| ) |
| |
| |
| def GenTests(api): |
| revision = "75b05681239cb309a23fcb4f8864f177e5aa62da" |
| builders = { |
| "linux-amd64": ["linux-x64-builder"], |
| "mac-amd64": ["mac-x64-builder"], |
| } |
| |
| for platform in ("linux", "mac"): |
| for package in ("rust",): |
| for arch in ("intel", "arm") if platform == "linux" else ("intel",): |
| yield ( |
| api.status_check.test("%s_%s_%s" % (package, arch, platform)) |
| + api.platform.name(platform) |
| + api.platform.arch(arch) |
| + api.properties( |
| package=package, |
| builders=builders, |
| ) |
| + api.git.get_remote_branch_head("git ls-remote", revision) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| ) |
| yield ( |
| api.status_check.test("rust_intel_linux_full") |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + api.properties( |
| package="rust", |
| builders=builders, |
| full_test_suite=True, |
| ) |
| + api.git.get_remote_branch_head("git ls-remote", revision) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| ) |
| yield ( |
| api.status_check.test("failing_tests") |
| + api.platform.name("linux") |
| + api.platform.arch("intel") |
| + api.properties(package="rust", builders=builders) |
| + api.git.get_remote_branch_head("git ls-remote", revision) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| + api.step_data("generate runtime spec", stdout=api.json.output([])) |
| + api.step_data("rust test x86_64-fuchsia", retcode=1) |
| ) |
