| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building Clang toolchain.""" |
| |
| import contextlib |
| import re |
| |
| from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest |
| from PB.recipes.fuchsia.contrib.clang_toolchain import InputProperties, LTOMode |
| |
| GIT_VERSION = "version:2@2.35.1.chromium.8" |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/cas_util", |
| "fuchsia/cipd_util", |
| "fuchsia/cmake", |
| "fuchsia/git", |
| "fuchsia/git_checkout", |
| "fuchsia/go", |
| "fuchsia/goma", |
| "fuchsia/gsutil", |
| "fuchsia/macos_sdk", |
| "fuchsia/ninja", |
| "fuchsia/platform_util", |
| "fuchsia/python3", |
| "fuchsia/rbe", |
| "fuchsia/tensorflow", |
| "fuchsia/toolchain", |
| "fuchsia/windows_sdk", |
| "fuchsia/zlib", |
| "fuchsia/zstd", |
| "recipe_engine/archive", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/platform", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/resultdb", |
| "recipe_engine/step", |
| "recipe_engine/url", |
| ] |
| |
| TARGET_TO_ARCH = { |
| "x64": "x86_64", |
| "arm64": "aarch64", |
| } |
| TARGETS = TARGET_TO_ARCH.keys() |
| |
| LIBXML2_GIT = "https://fuchsia.googlesource.com/third_party/libxml2" |
| BORINGSSL_GIT = "https://boringssl.googlesource.com/boringssl" |
| CURL_GIT = "https://fuchsia.googlesource.com/third_party/curl" |
| YASM_GIT = "https://fuchsia.googlesource.com/third_party/yasm" |
| CPP_HTTPLIB_GIT = "https://llvm.googlesource.com/third_party/cpp-httplib" |
| NCURSES_GIT = "https://llvm.googlesource.com/third_party/ncurses" |
| LIBEDIT_GIT = "https://llvm.googlesource.com/third_party/libedit" |
| LIBFFI_GIT = "https://llvm.googlesource.com/third_party/libffi" |
| CPYTHON_GIT = "https://llvm.googlesource.com/third_party/cpython" |
| CPYTHON_VERSION = "3.11.4" |
| |
| # TODO(https://fxbug.dev/327507421): Remove this temporary SDK version pin when |
| # we have a concrete solution. |
| FUCHSIA_SDK_VERSION = "f18" |
| |
| CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com" |
| |
| # TODD(fxbug.dev/91157): Restore the file name once |
| # path length issue is properly fixed. |
| RESULTDB_JSON = "r.j" |
| |
| NINJA_LOG = ".ninja_log" |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps( |
| api, |
| props, |
| ): |
| # default values |
| if not props.repository: |
| props.repository = "https://llvm.googlesource.com/llvm-project" |
| if not props.revision: |
| props.revision = "refs/heads/main" |
| if not props.artifact_gcs_bucket: |
| props.artifact_gcs_bucket = "fuchsia-artifacts" |
| |
| use_goma = ( |
| not api.platform.arch == "arm" and api.platform.bits == 64 |
| ) and not api.platform.is_win |
| |
| if props.use_rbe: |
| ninja_jobs = api.goma.jobs # doesn't really use Goma |
| # Valid choices for AbsolutePathPolicy for cmake-based builds are: |
| # RELATIVIZE: rewrite commands using relative paths for remote |
| # execution. |
| # ALLOW: Force the remote environment to look like local paths. |
| remote_build_context = api.rbe( |
| absolute_path_policy=api.rbe.AbsolutePathPolicy.ALLOW |
| ) |
| elif use_goma: |
| ninja_jobs = api.goma.jobs |
| remote_build_context = api.goma() |
| else: |
| ninja_jobs = api.platform.cpu_count |
| remote_build_context = contextlib.nullcontext() |
| |
| host_platform = api.cipd_util.platform_name |
| target_platform = props.platform or host_platform |
| |
| use_breakpad = host_platform == "linux-amd64" |
| # TODO: Make build_lldb an infra configurable option |
| build_lldb = True |
| should_build_ncurses = ( |
| build_lldb |
| and target_platform != "mac-arm64" |
| and target_platform != "windows-amd64" |
| ) |
| should_build_cpython = ( |
| build_lldb |
| and host_platform == target_platform |
| and target_platform != "windows-amd64" |
| ) |
| should_fetch_cpython = ( |
| build_lldb |
| and host_platform == target_platform |
| and target_platform == "windows-amd64" |
| ) |
| should_run_lldb_test = should_build_cpython |
| |
| manifest = Manifest() |
| use_ninjatrace = False |
| |
| with api.step.nest("ensure_packages"): |
| with api.context(infra_steps=True): |
| cipd_dir = api.path.start_dir / "cipd" |
| pkgs = api.cipd.EnsureFile() |
| # Use Linux x64 SDK on all platforms. We only need the sysroot, we |
| # don't use any tools so the host architecture is irrelevant. |
| pkgs.add_package("fuchsia/sdk/core/linux-amd64", FUCHSIA_SDK_VERSION, "sdk") |
| pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration") |
| # TODO(fxbug.dev/87518) Unify cmake and ninja revisions after test failures are fixed. |
| if api.platform.is_win: |
| # boringssl and clang rely on POSIX tools provided in the |
| # "git for windows" package, but they are not in PATH. |
| # Bootstrap git again so these tools can be added to PATH. |
| pkgs.add_package("infra/3pp/tools/git/${platform}", GIT_VERSION) |
| if not api.platform.is_win: |
| # Experimental windows runtime cross compilation under Linux. |
| win_sdk_dir = api.windows_sdk.ensure_sdk() |
| pkgs.add_package( |
| "fuchsia/third_party/sysroot/linux", |
| "integration", |
| "linux", |
| ) |
| pkgs.add_package( |
| "fuchsia/third_party/sysroot/focal", |
| "git_revision:fa7a5a9710540f30ff98ae48b62f2cdf72ed2acd", |
| "focal", |
| ) |
| if not target_platform in [ |
| "linux-arm64", |
| "mac-arm64", |
| ]: |
| pkgs.add_package( |
| f"fuchsia/model/inlining/{target_platform}", |
| "git_revision:0ee1ce61a844b8fd285ff1a2c47c4b8690fd2b7d", |
| "model", |
| ) |
| if use_breakpad: |
| pkgs.add_package( |
| "fuchsia/tools/breakpad/${platform}", "integration", "breakpad" |
| ) |
| if api.platform.arch != "arm" and not api.platform.is_win: |
| use_ninjatrace = True |
| pkgs.add_package( |
| "fuchsia/tools/ninjatrace/${platform}", "latest", "ninjatrace" |
| ) |
| if build_lldb: |
| if should_build_cpython or should_build_ncurses: |
| # There is no make for Windows in CIPD. |
| # Current Windows LLDB setup also doesn't need GNU make. |
| pkgs.add_package( |
| "fuchsia/third_party/make/${platform}", "version:4.3" |
| ) |
| pkgs.add_package("infra/3pp/tools/swig/${platform}", "latest") |
| if api.platform.is_linux: |
| # TODO: Remove this once we update compiler-rt to no longer assume binutils. |
| pkgs.add_package( |
| "fuchsia/third_party/binutils-gdb/${platform}", |
| "git_revision:9030a82d6f700e03ab143f0d002e9f21ae2fd52f", |
| "binutils-gdb", |
| ) |
| |
| ensured = api.cipd.ensure(cipd_dir, pkgs) |
| for subdir, pins in ensured.items(): |
| directory = manifest.directories[str(cipd_dir / subdir)] |
| directory.cipd_server_host = CIPD_SERVER_HOST |
| for pin in pins: |
| directory.cipd_package[pin.package].instance_id = pin.instance_id |
| |
| with api.step.nest("update sdk"): |
| sdk_dir = api.path.start_dir / "sdk" |
| api.file.copytree("copy sdk", cipd_dir / "sdk", sdk_dir) |
| api.file.chmod( |
| "chmod pkg/sysroot/meta.json", |
| sdk_dir.joinpath("pkg", "sysroot", "meta.json"), |
| "0644", |
| ) |
| |
| staging_dir = api.path.start_dir |
| pkg_dir = staging_dir / "llvm_install" |
| api.file.ensure_directory("create pkg dir", pkg_dir) |
| if build_lldb: |
| lldb_pkg_dir = staging_dir / "lldb_install" |
| api.file.ensure_directory("create lldb pkg dir", lldb_pkg_dir) |
| |
| llvm_dir, revision = api.git_checkout(props.repository, fallback_ref=props.revision) |
| git_checkout = manifest.directories[str(llvm_dir)].git_checkout |
| git_checkout.repo_url = props.repository |
| git_checkout.revision = revision |
| |
| with api.context(cwd=llvm_dir): |
| for revert in props.reverts: |
| api.git.revert(revert) |
| |
| target_triple = api.toolchain.PLATFORM_TO_TRIPLE[target_platform] |
| host_triple = api.toolchain.PLATFORM_TO_TRIPLE[host_platform] |
| cas_digest = "" |
| clang_version = "" |
| if api.platform.is_win: |
| # Add git's tools to PATH. |
| env_prefixes = { |
| "PATH": [cipd_dir.joinpath("usr", "bin"), cipd_dir.joinpath("bin")] |
| } |
| else: |
| env_prefixes = {"PATH": [cipd_dir / "bin"]} |
| |
| with api.macos_sdk( |
| kind="ios" |
| ), api.windows_sdk(), remote_build_context, api.context(env_prefixes=env_prefixes): |
| if api.platform.name == "linux": |
| target_sysroot = host_sysroot = cipd_dir / "linux" |
| elif api.platform.name == "mac": |
| # TODO(fxbug.dev/3043): Eventually use our own hermetic sysroot as for Linux. |
| target_sysroot = host_sysroot = api.macos_sdk.sysroot |
| elif api.platform.name == "win": |
| target_sysroot = host_sysroot = api.windows_sdk.sdk_dir |
| else: # pragma: no cover |
| assert False, "unsupported platform" |
| |
| arguments = { |
| "target_triple": target_triple, |
| "host_triple": host_triple, |
| "target_sysroot": target_sysroot, |
| "host_sysroot": host_sysroot, |
| "linux_sysroot": cipd_dir / "linux", |
| "fuchsia_sdk": sdk_dir, |
| "ninja": api.ninja.path, |
| } |
| arguments.update( |
| { |
| "win": { |
| "cc": cipd_dir.joinpath("bin", "clang-cl.exe"), |
| "cxx": cipd_dir.joinpath("bin", "clang-cl.exe"), |
| "ar": cipd_dir.joinpath("bin", "llvm-lib.exe"), |
| "ld": cipd_dir.joinpath("bin", "lld-link.exe"), |
| "mt": cipd_dir.joinpath("bin", "llvm-mt.exe"), |
| "nm": cipd_dir.joinpath("bin", "llvm-nm.exe"), |
| "objcopy": cipd_dir.joinpath("bin", "llvm-objcopy.exe"), |
| "objdump": cipd_dir.joinpath("bin", "llvm-objdump.exe"), |
| "ranlib": cipd_dir.joinpath("bin", "llvm-ranlib.exe"), |
| "rc": cipd_dir.joinpath("bin", "llvm-rc.exe"), |
| "readelf": cipd_dir.joinpath("bin", "llvm-readelf.exe"), |
| "strip": cipd_dir.joinpath("bin", "llvm-strip.exe"), |
| }, |
| "mac": { |
| "cc": cipd_dir.joinpath("bin", "clang"), |
| "cxx": cipd_dir.joinpath("bin", "clang++"), |
| "ar": cipd_dir.joinpath("bin", "llvm-ar"), |
| "ld": "/usr/bin/ld", |
| "libtool": cipd_dir.joinpath("bin", "llvm-libtool-darwin"), |
| "lipo": cipd_dir.joinpath("bin", "llvm-lipo"), |
| "nm": cipd_dir.joinpath("bin", "llvm-nm"), |
| "objcopy": cipd_dir.joinpath("bin", "llvm-objcopy"), |
| "objdump": cipd_dir.joinpath("bin", "llvm-objdump"), |
| "ranlib": cipd_dir.joinpath("bin", "llvm-ranlib"), |
| "readelf": cipd_dir.joinpath("bin", "llvm-readelf"), |
| "strip": cipd_dir.joinpath("bin", "llvm-strip"), |
| }, |
| "linux": { |
| "cc": cipd_dir.joinpath("bin", "clang"), |
| "cxx": cipd_dir.joinpath("bin", "clang++"), |
| "ar": cipd_dir.joinpath("bin", "llvm-ar"), |
| "ld": cipd_dir.joinpath("bin", "ld.lld"), |
| "nm": cipd_dir.joinpath("bin", "llvm-nm"), |
| "objcopy": cipd_dir.joinpath("bin", "llvm-objcopy"), |
| "objdump": cipd_dir.joinpath("bin", "llvm-objdump"), |
| "ranlib": cipd_dir.joinpath("bin", "llvm-ranlib"), |
| "readelf": cipd_dir.joinpath("bin", "llvm-readelf"), |
| "strip": cipd_dir.joinpath("bin", "llvm-strip"), |
| }, |
| }[api.platform.name] |
| ) |
| |
| if props.use_rbe: |
| arguments.update( |
| {"compiler_wrapper": api.rbe.cxx_compiler_wrapper()} |
| # TODO(https://fxbug.dev/107610): prefer pass a multi-token |
| # wrapper directly, joining with ';'. |
| # {"compiler_wrapper": ";".join(api.rbe.cxx_compiler_wrapper_command())} |
| ) |
| elif use_goma: |
| arguments.update({"compiler_wrapper": api.goma.goma_dir / "gomacc"}) |
| |
| options = [ |
| "-GNinja", |
| "-DCMAKE_MAKE_PROGRAM={ninja}", |
| "-DCMAKE_INSTALL_PREFIX=", |
| "-DCMAKE_C_COMPILER={cc}", |
| "-DCMAKE_CXX_COMPILER={cxx}", |
| "-DCMAKE_ASM_COMPILER={cc}", |
| f"-DCLANG_REPOSITORY_STRING={props.repository}", |
| ] |
| if use_goma or props.use_rbe: |
| options.extend( |
| [ |
| "-DCMAKE_C_COMPILER_LAUNCHER={compiler_wrapper}", |
| "-DCMAKE_CXX_COMPILER_LAUNCHER={compiler_wrapper}", |
| "-DCMAKE_ASM_COMPILER_LAUNCHER={compiler_wrapper}", |
| ] |
| ) |
| |
| options.extend( |
| { |
| "linux": [ |
| "-DCMAKE_AR={ar}", |
| "-DCMAKE_LINKER={ld}", |
| "-DCMAKE_NM={nm}", |
| "-DCMAKE_OBJCOPY={objcopy}", |
| "-DCMAKE_OBJDUMP={objdump}", |
| "-DCMAKE_RANLIB={ranlib}", |
| "-DCMAKE_READELF={readelf}", |
| "-DCMAKE_STRIP={strip}", |
| ], |
| "mac": [ |
| "-DCMAKE_LIBTOOL={libtool}", |
| "-DCMAKE_LIPO={lipo}", |
| ], |
| "win": [ |
| "-DCMAKE_AR={ar}", |
| "-DCMAKE_LINKER={ld}", |
| "-DCMAKE_NM={nm}", |
| "-DCMAKE_OBJCOPY={objcopy}", |
| "-DCMAKE_OBJDUMP={objdump}", |
| "-DCMAKE_RANLIB={ranlib}", |
| "-DCMAKE_READELF={readelf}", |
| "-DCMAKE_STRIP={strip}", |
| # TODO(phosek): reenable once we update the host toolchain |
| # "-DCMAKE_RC_COMPILER={rc}", |
| # "-DCMAKE_MT={mt}", |
| ], |
| }[api.platform.name] |
| ) |
| |
| if api.platform.is_mac and not props.enable_lld: |
| options.extend(["-DCMAKE_LINKER={ld}"]) |
| |
| options.extend(["-DCMAKE_SYSROOT={target_sysroot}"]) |
| |
| # TODO(phosek): consider moving these to a cache file |
| platform_options = [] |
| if api.platform.is_mac: |
| platform_options.extend(["-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13"]) |
| platform_options.extend( |
| [ |
| f"-DCMAKE_{mode}_LINKER_FLAGS=-nostdlib++ {cipd_dir.joinpath('lib', 'libc++.a')}" |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| arguments.update( |
| { |
| "cflags": "-mmacosx-version-min=10.13", |
| "cxxflags": "-mmacosx-version-min=10.13", |
| "ldflags": "-nostdlib++ " |
| + str(api.path.start_dir.joinpath("cipd", "lib", "libc++.a")), |
| } |
| ) |
| else: |
| arguments.update({"cflags": "", "cxxflags": "", "ldflags": ""}) |
| |
| if api.platform.is_win: |
| platform_options.extend( |
| [ |
| "-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded", |
| # The above flag was added in CMake 3.15, but some |
| # dependencies have earlier minimum versions. This policy |
| # switch applies the above flag to these as well. |
| "-DCMAKE_POLICY_DEFAULT_CMP0091=NEW", |
| ] |
| ) |
| platform_options_target = platform_options.copy() |
| if host_triple != target_triple: |
| # TODO(phosek): Switch to api.platform_util and avoid string split. |
| (os, arch) = target_platform.split("-") |
| system_name = os.replace("mac", "darwin").title() |
| system_processor = arch.replace("amd64", "x86_64").replace( |
| "arm64", "aarch64" |
| ) |
| toolchain_file = api.path.start_dir / "ToolChain.cmake" |
| api.file.write_text( |
| "write CMake toolchain file", |
| toolchain_file, |
| "\n".join( |
| [ |
| f"set(CMAKE_SYSTEM_NAME {system_name})", |
| f"set(CMAKE_SYSTEM_PROCESSOR {system_processor})", |
| f"set(CMAKE_C_COMPILER_TARGET {target_triple})", |
| f"set(CMAKE_CXX_COMPILER_TARGET {target_triple})", |
| f"set(CMAKE_ASM_COMPILER_TARGET {target_triple})", |
| "", |
| ] |
| ), |
| ) |
| platform_options_target.append(f"-DCMAKE_TOOLCHAIN_FILE={toolchain_file}") |
| |
| with api.step.nest("zlib"): |
| zlib_build_dir_target = staging_dir / "zlib_build_target" |
| api.file.ensure_directory("create zlib_build_dir", zlib_build_dir_target) |
| zlib_install_dir_target = staging_dir / "zlib_install_target" |
| api.file.ensure_directory( |
| "create zlib_install_dir", zlib_install_dir_target |
| ) |
| api.zlib.build( |
| cmake_extra_args=[ |
| option.format(**arguments) |
| for option in options |
| + platform_options_target |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET. |
| f"-DCMAKE_{lang}_FLAGS=--target={target_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ] |
| + [ |
| # TODO: once we no longer build libLTO, we can drop |
| # this. |
| "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", |
| ] |
| ], |
| platform=api.platform_util.platform(target_platform), |
| ninja_jobs=ninja_jobs, |
| build_dir=zlib_build_dir_target, |
| install_dir=zlib_install_dir_target, |
| ) |
| |
| use_zlib_options_target = [ |
| f"-DZLIB_INCLUDE_DIR={zlib_install_dir_target / 'include'}", |
| f"-DZLIB_LIBRARY={zlib_install_dir_target / 'lib' / ('zlibstatic.lib' if api.platform.is_win else 'libz.a')}", |
| ] |
| |
| if host_triple != target_triple: |
| zlib_build_dir_host = staging_dir / "zlib_build_host" |
| api.file.ensure_directory("create zlib_build_dir", zlib_build_dir_host) |
| zlib_install_dir_host = staging_dir / "zlib_install_host" |
| api.file.ensure_directory( |
| "create zlib_install_dir", zlib_install_dir_host |
| ) |
| api.zlib.build( |
| cmake_extra_args=[ |
| option.format(**arguments) |
| for option in options |
| + platform_options |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET. |
| f"-DCMAKE_{lang}_FLAGS=--target={host_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ] |
| + [ |
| # TODO: once we no longer build libLTO, we can drop |
| # this. |
| "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", |
| ] |
| ], |
| platform=api.platform_util.platform(host_platform), |
| ninja_jobs=ninja_jobs, |
| build_dir=zlib_build_dir_host, |
| install_dir=zlib_install_dir_host, |
| ) |
| use_zlib_options_host = [ |
| f"-DZLIB_INCLUDE_DIR={zlib_install_dir_host.joinpath('include')}", |
| f"-DZLIB_LIBRARY={zlib_install_dir_host.joinpath('lib', 'zlibstatic.lib' if api.platform.is_win else 'libz.a')}", |
| ] |
| |
| with api.step.nest("zstd"): |
| zstd_build_dir = staging_dir / "zstd_build" |
| api.file.ensure_directory("create zstd", zstd_build_dir) |
| zstd_install_dir = staging_dir / "zstd_install" |
| api.file.ensure_directory("create zstd_install_dir", zstd_install_dir) |
| api.zstd.build( |
| cmake_extra_args=[ |
| option.format(**arguments) |
| for option in options |
| + platform_options_target |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET |
| f"-DCMAKE_{lang}_FLAGS=--target={target_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ] |
| + [ |
| "-DZSTD_BUILD_SHARED=OFF", |
| # TODO: once we no longer build libLTO, we can drop this |
| "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", |
| ] |
| ], |
| ninja_jobs=ninja_jobs, |
| build_dir=zstd_build_dir, |
| install_dir=zstd_install_dir, |
| ) |
| |
| with api.step.nest("libxml2"): |
| libxml2_install_dir_target = staging_dir / "libxml2_install_target" |
| api.file.ensure_directory( |
| "create libxml2_install_dir", libxml2_install_dir_target |
| ) |
| libxml2_cmake_dir_target, libxml2_dir_target = build_libxml2( |
| api, |
| options |
| + platform_options_target |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET |
| f"-DCMAKE_{lang}_FLAGS=--target={target_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ] |
| + use_zlib_options_target |
| + [ |
| # TODO: once we no longer build libLTO, we can drop this |
| "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", |
| ], |
| arguments, |
| libxml2_install_dir_target, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| dir_suffix="_target", |
| ) |
| if host_triple != target_triple: |
| libxml2_install_dir_host = staging_dir / "libxml2_install_host" |
| api.file.ensure_directory( |
| "create libxml2_install_dir for host", libxml2_install_dir_host |
| ) |
| libxml2_cmake_dir_host, _ = build_libxml2( |
| api, |
| options |
| + platform_options |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET |
| f"-DCMAKE_{lang}_FLAGS=--target={host_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ] |
| + use_zlib_options_host |
| + [ |
| # TODO: once we no longer build libLTO, we can drop this |
| "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", |
| ], |
| arguments, |
| libxml2_install_dir_host, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| dir_suffix="_host", |
| ) |
| |
| if api.platform.is_win: |
| with api.step.nest("yasm"): |
| yasm_install_dir = staging_dir / "yasm_install" |
| api.file.ensure_directory("create yasm install dir", yasm_install_dir) |
| build_yasm( |
| api, |
| options |
| + platform_options_target |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET |
| f"-DCMAKE_{lang}_FLAGS=--target={target_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ], |
| arguments, |
| yasm_install_dir, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| ) |
| |
| with api.step.nest("boringssl"): |
| boringssl_install_dir = staging_dir / "boringssl_install" |
| api.file.ensure_directory( |
| "create boringssl_install_dir", boringssl_install_dir |
| ) |
| nasm_option = [] |
| if api.platform.is_win: |
| nasm_option = [ |
| f"-DCMAKE_ASM_NASM_COMPILER={yasm_install_dir.joinpath('bin', 'yasm.exe')}" |
| ] |
| boringssl_pic_option = [] |
| if build_lldb: |
| boringssl_pic_option = ["-DCMAKE_POSITION_INDEPENDENT_CODE=ON"] |
| boringssl_cmake_dir = build_boringssl( |
| api, |
| options |
| + platform_options_target |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET |
| f"-DCMAKE_{lang}_FLAGS=--target={target_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ] |
| + [f"-DGO_EXECUTABLE={api.go.go_root.joinpath('bin', 'go')}"] |
| + nasm_option |
| + boringssl_pic_option, |
| arguments, |
| boringssl_install_dir, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| ) |
| |
| with api.step.nest("curl"): |
| curl_install_dir = staging_dir / "curl_install" |
| api.file.ensure_directory("create curl install dir", curl_install_dir) |
| curl_cmake_dir = build_curl( |
| api, |
| options |
| + platform_options_target |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET |
| f"-DCMAKE_{lang}_FLAGS=--target={target_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ] |
| + [ |
| "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON", |
| f"-DOpenSSL_ROOT={boringssl_cmake_dir}", |
| ] |
| + use_zlib_options_target, |
| arguments, |
| curl_install_dir, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| ) |
| |
| with api.step.nest("cpp-httplib"): |
| cpp_httplib_install_dir = staging_dir / "cpp_httplib_install" |
| api.file.ensure_directory( |
| "create cpp-httplib install dir", cpp_httplib_install_dir |
| ) |
| cpp_httplib_cmake_dir = build_cpp_httplib( |
| api, |
| options |
| + platform_options_target |
| + [ |
| # TODO: once we upgrade to CMake 3.20, we can use |
| # CMAKE_{C,CXX,ASM}_COMPILER_TARGET |
| f"-DCMAKE_{lang}_FLAGS=--target={target_triple}" |
| for lang in ["C", "CXX", "ASM"] |
| ] |
| + [ |
| "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON", |
| ] |
| + use_zlib_options_target, |
| arguments, |
| cpp_httplib_install_dir, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| ) |
| |
| if build_lldb: |
| if should_build_ncurses: |
| with api.step.nest("ncurses"): |
| ncurses_install_dir = staging_dir / "ncurses_install" |
| api.file.ensure_directory( |
| "create ncurses install dir", ncurses_install_dir |
| ) |
| build_ncurses( |
| api, |
| arguments, |
| ncurses_install_dir, |
| manifest, |
| ) |
| with api.step.nest("libedit"): |
| libedit_install_dir = staging_dir / "libedit_install" |
| api.file.ensure_directory( |
| "create libedit install dir", libedit_install_dir |
| ) |
| build_libedit( |
| api, |
| arguments, |
| ncurses_install_dir, |
| libedit_install_dir, |
| manifest, |
| ) |
| if should_fetch_cpython: |
| cpython_install_dir = lldb_pkg_dir / "python3" |
| api.file.ensure_directory( |
| "create cpython install dir", cpython_install_dir |
| ) |
| fetch_and_extract_windows_python( |
| api, CPYTHON_VERSION, cpython_install_dir |
| ) |
| |
| if should_build_cpython: |
| with api.step.nest("libffi"): |
| libffi_install_dir = staging_dir / "libffi_install" |
| api.file.ensure_directory( |
| "create libffi install dir", libffi_install_dir |
| ) |
| build_libffi( |
| api, |
| arguments, |
| libffi_install_dir, |
| manifest, |
| ) |
| with api.step.nest("cpython"): |
| zlib_install_dir = ( |
| zlib_install_dir_host |
| if host_triple != target_triple |
| else zlib_install_dir_target |
| ) |
| cpython_install_dir = lldb_pkg_dir / "python3" |
| api.file.ensure_directory( |
| "create cpython install dir", cpython_install_dir |
| ) |
| build_cpython( |
| api, |
| arguments, |
| libedit_install_dir, |
| libffi_install_dir, |
| ncurses_install_dir, |
| zlib_install_dir, |
| cpython_install_dir, |
| manifest, |
| ) |
| if api.platform.is_mac: |
| with api.step.nest("lldb keychain"): |
| make_lldb_keychain(api) |
| |
| api.file.write_proto( |
| "source manifest", pkg_dir / "source_manifest.json", manifest, "JSONPB" |
| ) |
| |
| # build clang+llvm |
| build_dir = staging_dir / "llvm_build" |
| api.file.ensure_directory("create llvm build dir", build_dir) |
| |
| # The LLDB Python host looks for libraries relative to the executable, |
| # so copy the libraries from the install directory into the appropriate |
| # relative paths from binaries used during testing. |
| if build_lldb: |
| lldb_build_dir = build_dir |
| if props.do_2stage: |
| lldb_build_dir = lldb_build_dir.joinpath( |
| "tools", "clang", "stage2-bins" |
| ) |
| |
| if should_build_cpython or should_fetch_cpython: |
| api.file.copytree( |
| "copy python libraries to build dir", |
| cpython_install_dir, |
| lldb_build_dir / "python3", |
| ) |
| api.file.copytree( |
| "copy python libraries to unit test dir", |
| cpython_install_dir, |
| lldb_build_dir.joinpath( |
| "tools", "lldb", "unittests", "ScriptInterpreter", "python3" |
| ), |
| ) |
| |
| arguments.update( |
| { |
| "BOOTSTRAP_": "", |
| "STAGE2_": "STAGE2_", |
| "stage2_": "stage2-", |
| "_stage2": "", |
| } |
| if props.do_2stage |
| else {"BOOTSTRAP_": "", "STAGE2_": "", "stage2_": "", "_stage2": "-stage2"} |
| ) |
| |
| llvm_projects = ["clang", "clang-tools-extra", "lld", "llvm", "polly"] |
| llvm_runtimes = ["compiler-rt", "libcxx", "libcxxabi", "libunwind"] |
| |
| options.extend( |
| [ |
| "-D{BOOTSTRAP_}LLVM_ENABLE_ZLIB=FORCE_ON", |
| "-D{BOOTSTRAP_}ZLIB_INCLUDE_DIR=%s" |
| % zlib_install_dir_target.joinpath("include"), |
| "-D{BOOTSTRAP_}ZLIB_LIBRARY=%s" |
| % zlib_install_dir_target.joinpath( |
| "lib", "zlibstatic.lib" if api.platform.is_win else "libz.a" |
| ), |
| "-D{BOOTSTRAP_}LLVM_ENABLE_ZSTD=FORCE_ON", |
| "-D{BOOTSTRAP_}zstd_DIR=%s" |
| % zstd_install_dir.joinpath("lib", "cmake", "zstd"), |
| "-D{BOOTSTRAP_}LLVM_ENABLE_LIBXML2=FORCE_ON", |
| # Note that the LibXml2 spelling has to match the spelling used in |
| # https://github.com/llvm/llvm-project/blob/cf54424a/llvm/cmake/config-ix.cmake#L144 |
| "-D{BOOTSTRAP_}LibXml2_ROOT=%s" % libxml2_cmake_dir_target, |
| "-D{BOOTSTRAP_}LLVM_ENABLE_CURL=FORCE_ON", |
| "-D{BOOTSTRAP_}CURL_ROOT=%s" % curl_cmake_dir, |
| "-D{BOOTSTRAP_}OpenSSL_ROOT=%s" % boringssl_cmake_dir, |
| "-D{BOOTSTRAP_}LLVM_ENABLE_HTTPLIB=FORCE_ON", |
| "-D{BOOTSTRAP_}httplib_ROOT=%s" % cpp_httplib_cmake_dir, |
| "-D{BOOTSTRAP_}CMAKE_FIND_PACKAGE_PREFER_CONFIG=ON", |
| ] |
| ) |
| |
| if not api.platform.is_win and host_triple == target_triple: |
| options.extend( |
| [ |
| "-DLLVM_WINSYSROOT=%s" % win_sdk_dir, |
| "-DLLVM_VFSOVERLAY=%s" |
| % win_sdk_dir.joinpath("llvm-vfsoverlay.yaml"), |
| ] |
| ) |
| if api.platform.name == "linux": |
| options.extend( |
| [ |
| # Unprefixed flags are used by the first stage compiler. |
| f"-DCMAKE_{mode}_LINKER_FLAGS=-static-libstdc++" |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| if props.do_2stage: |
| # TODO(fxbug.dev/81937) |
| # This is a temporary work around to resolve the out of memory issue that |
| # linux-arm64 builders running into, and should be removed when we implement |
| # support for fat LTO in Clang. |
| if api.platform.arch == "arm": |
| options.extend( |
| [ |
| "-D{STAGE2_}LLVM_PARALLEL_LINK_JOBS=8", |
| ] |
| ) |
| |
| elif api.platform.name == "mac": |
| if props.do_2stage: |
| options.extend( |
| [ |
| # BOOTSTRAP_ prefixed flags are passed to the second stage compiler. |
| "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-nostdlib++ %s" |
| % (mode, build_dir.joinpath("lib", "libc++.a")) |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| + [ |
| # Unprefixed flags are used by the first stage compiler. |
| f"-DCMAKE_{mode}_LINKER_FLAGS=-nostdlib++ {cipd_dir.joinpath('lib', 'libc++.a')}" |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| + (["-D{STAGE2_}CMAKE_LINKER={ld}"] if not props.enable_lld else []) |
| ) |
| else: |
| options.extend( |
| [ |
| # BOOTSTRAP_ prefixed flags are passed to the second stage compiler. |
| "-DCMAKE_%s_LINKER_FLAGS=-nostdlib++ %s" |
| % (mode, cipd_dir.joinpath("lib", "libc++.a")) |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| |
| if host_triple != target_triple: |
| arguments.update( |
| { |
| "BOOTSTRAP_": "BOOTSTRAP_", |
| } |
| ) |
| options.extend( |
| [ |
| "-D{BOOTSTRAP_}CMAKE_SYSTEM_NAME=%s" % system_name, |
| "-D{BOOTSTRAP_}CMAKE_SYSTEM_PROCESSOR=%s" % system_processor, |
| "-D{BOOTSTRAP_}CMAKE_C_COMPILER_TARGET={target_triple}", |
| "-D{BOOTSTRAP_}CMAKE_CXX_COMPILER_TARGET={target_triple}", |
| "-D{BOOTSTRAP_}CMAKE_ASM_COMPILER_TARGET={target_triple}", |
| "-D{BOOTSTRAP_}LLVM_DEFAULT_TARGET_TRIPLE={target_triple}", |
| "-DLLVM_ENABLE_ZLIB=FORCE_ON", |
| "-DZLIB_INCLUDE_DIR=%s" % zlib_install_dir_host.joinpath("include"), |
| "-DZLIB_LIBRARY=%s" |
| % zlib_install_dir_host.joinpath( |
| "lib", "zlibstatic.lib" if api.platform.is_win else "libz.a" |
| ), |
| "-DLLVM_ENABLE_LIBXML2=FORCE_ON", |
| # Note that the LibXml2 spelling has to match the spelling used in |
| # https://github.com/llvm/llvm-project/blob/cf54424a/llvm/cmake/config-ix.cmake#L144 |
| "-DLibXml2_ROOT=%s" % libxml2_cmake_dir_host, |
| "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON", |
| ] |
| ) |
| |
| # STAGE2_ prefixed flags are passed to the second stage by the first stage build. |
| |
| options.extend( |
| [ |
| "-D{STAGE2_}LINUX_aarch64-unknown-linux-gnu_SYSROOT={linux_sysroot}", |
| "-D{STAGE2_}LINUX_armv7-unknown-linux-gnueabihf_SYSROOT={linux_sysroot}", |
| "-D{STAGE2_}LINUX_i386-unknown-linux-gnu_SYSROOT={linux_sysroot}", |
| "-D{STAGE2_}LINUX_riscv64-unknown-linux-gnu_SYSROOT=%s" |
| % cipd_dir.joinpath("focal"), |
| "-D{STAGE2_}LINUX_x86_64-unknown-linux-gnu_SYSROOT={linux_sysroot}", |
| "-D{STAGE2_}FUCHSIA_SDK={fuchsia_sdk}", |
| "-D{STAGE2_}LLVM_LIT_ARGS=--resultdb-output=%s -v" % RESULTDB_JSON, |
| ] |
| ) |
| |
| cross_native_flags = [ |
| "-DCMAKE_C_COMPILER={cc}", |
| "-DCMAKE_CXX_COMPILER={cxx}", |
| "-DCMAKE_%sSYSROOT={host_sysroot}" |
| % ("OSX_" if api.platform.is_mac else ""), |
| ] |
| |
| if api.platform.is_mac: |
| cross_native_flags.extend( |
| [ |
| f"-DCMAKE_{mode}_LINKER_FLAGS=-nostdlib++ {cipd_dir.joinpath('lib', 'libc++.a')}" |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| |
| # TODO(fxbug.dev/127486): The stage2 build of the host tools in a cross compile doesn't |
| # correctly use CMAKE_C{,XX}_COMPILER. This solution isn't ideal because it uses the |
| # compiler from cipd and not the one we just built in stage1, but this is better than it |
| # using a system compiler that no longer exists. See https://reviews.llvm.org/D126313#3543242. |
| options.append( |
| "-D{STAGE2_}CROSS_TOOLCHAIN_FLAGS_NATIVE=" + ";".join(cross_native_flags) |
| ) |
| |
| options.append( |
| "-D{STAGE2_}LLVM_ENABLE_LTO=%s" % LTOMode.Name(props.lto_mode).title() |
| ) |
| |
| if props.enable_lld: |
| options.append("-D{STAGE2_}LLVM_ENABLE_LLD=%s" % props.enable_lld) |
| |
| if props.enable_assertions: |
| options.append( |
| "-D{STAGE2_}LLVM_ENABLE_ASSERTIONS=%s" % props.enable_assertions |
| ) |
| |
| if props.enable_backtraces: |
| options.append( |
| "-D{STAGE2_}LLVM_ENABLE_BACKTRACES=%s" % props.enable_backtraces |
| ) |
| |
| if not target_platform in [ |
| "linux-arm64", |
| "mac-arm64", |
| ]: |
| options.extend( |
| [ |
| "-D{STAGE2_}TENSORFLOW_AOT_PATH=%s" % api.tensorflow.path, |
| "-D{STAGE2_}LLVM_OVERRIDE_MODEL_HEADER_INLINERSIZEMODEL=%s" |
| % cipd_dir.joinpath("model", "InlinerSizeModel.h"), |
| "-D{STAGE2_}LLVM_OVERRIDE_MODEL_OBJECT_INLINERSIZEMODEL=%s" |
| % cipd_dir.joinpath("model", "InlinerSizeModel.o"), |
| "-D{STAGE2_}LLVM_RAEVICT_MODEL_PATH=none", |
| ] |
| ) |
| |
| if build_lldb: |
| if api.platform.is_mac: |
| if should_build_cpython: |
| python3_rpath = "@loader_path/../python3" |
| python3_libraries = ( |
| f"{cpython_install_dir}/lib/lib{PYTHON_MAJOR_MINOR}.dylib" |
| ) |
| options.append( |
| # This is default, but LLDB CMake looks for it in the |
| # system toolchain. We can't add a certificate to the |
| # system toolchain without GUI access, but the toolchain |
| # created earlier is on the search path, so setting this |
| # explicitly works despite the warning. |
| "-D{BOOTSTRAP_}LLDB_CODESIGN_IDENTITY=lldb_codesign", |
| ) |
| elif should_build_cpython or should_fetch_cpython: |
| python3_rpath = "$ORIGIN/../python3/lib" |
| if api.platform.is_win: |
| python3_libraries = f"{cpython_install_dir}/libs/{PYTHON_MAJOR_MINOR.replace('.','')}.lib" |
| else: |
| python3_libraries = ( |
| f"{cpython_install_dir}/lib/lib{PYTHON_MAJOR_MINOR}.so.1.0" |
| ) |
| options.extend( |
| [ |
| "-DFUCHSIA_ENABLE_LLDB=ON", |
| f"-DLLDB_ENABLE_PYTHON={'ON' if (should_build_cpython or should_fetch_cpython) else 'OFF'}", |
| ] |
| ) |
| if should_build_ncurses: |
| options.extend( |
| [ |
| "-D{BOOTSTRAP_}LLDB_ENABLE_CURSES=ON", |
| # Libedit requires ncurses for terminfo, but the wide |
| # character header check in LLDB does not include any |
| # library dependencies. |
| "-D{BOOTSTRAP_}LLDB_EDITLINE_USE_WCHAR=ON", |
| "-D{BOOTSTRAP_}CURSES_INCLUDE_DIRS=" |
| + f"{ncurses_install_dir}/include;{ncurses_install_dir}/include/ncurses", |
| "-D{BOOTSTRAP_}CURSES_LIBRARIES=" |
| + f"{ncurses_install_dir}/lib/libncurses.a", |
| "-D{BOOTSTRAP_}PANEL_LIBRARIES=" |
| + f"{ncurses_install_dir}/lib/libpanel.a", |
| "-D{BOOTSTRAP_}LLVM_ENABLE_TERMINFO=FORCE_ON", |
| "-D{BOOTSTRAP_}Terminfo_LIBRARIES=" |
| + f"{ncurses_install_dir}/lib/libncurses.a", |
| "-D{BOOTSTRAP_}LLVM_ENABLE_LIBEDIT=FORCE_ON", |
| "-D{BOOTSTRAP_}LLDB_ENABLE_LIBEDIT=ON", |
| "-D{BOOTSTRAP_}LibEdit_INCLUDE_DIRS=" |
| + f"{libedit_install_dir}/include", |
| "-D{BOOTSTRAP_}LibEdit_LIBRARIES=" |
| + f"{libedit_install_dir}/lib/libedit.a", |
| ] |
| ) |
| if should_build_cpython or should_fetch_cpython: |
| options.extend( |
| [ |
| "-DLLDB_EMBED_PYTHON_HOME=ON", |
| # Python directories are searched for in |
| # $PYTHONHOME/lib/python<MAJOR>.<MINOR>, relative to |
| # liblldb |
| "-DLLDB_PYTHON_HOME=../python3", |
| f"-DPython3_LIBRARIES={python3_libraries}", |
| f"-DPython3_RPATH={python3_rpath}", |
| "-D{BOOTSTRAP_}LLDB_TEST_USER_ARGS=--skip-category=pexpect", |
| ] |
| ) |
| if should_build_cpython: |
| options.extend( |
| [ |
| # This is the path to install the LLDB python packages to relative to the prefix. |
| # The autodetect script doesn't like the fake prefix and defaults to lib/python<x.xx>. |
| f"-DLLDB_PYTHON_RELATIVE_PATH=python3/lib/{PYTHON_MAJOR_MINOR}/site-packages", |
| f"-DPython3_EXECUTABLE={cpython_install_dir}/bin/python3", |
| f"-DPython3_INCLUDE_DIRS={cpython_install_dir}/include/{PYTHON_MAJOR_MINOR}", |
| ] |
| ) |
| if should_fetch_cpython: |
| options.extend( |
| [ |
| # This is the path to install the LLDB python packages to relative to the prefix. |
| # The autodetect script doesn't like the fake prefix and defaults to lib/python<x.xx>. |
| "-DLLDB_PYTHON_RELATIVE_PATH=python3/lib/site-packages", |
| f"-DPython3_EXECUTABLE={cpython_install_dir}/python.exe", |
| f"-DPython3_INCLUDE_DIRS={cpython_install_dir}/include", |
| ] |
| ) |
| if api.platform.is_win: |
| # On windows, cmake cannot find swig automatically even though it is |
| # placed in PATH. Use SWIG_DIR and SWIG_EXECUTABLE to explicitly to |
| # configure it. |
| options.extend( |
| [ |
| "-DSWIG_DIR=" + f"{cipd_dir / 'Lib'}", |
| "-DSWIG_EXECUTABLE=" + f"{cipd_dir / 'swig.exe'}", |
| ] |
| ) |
| |
| # Directory name for storing clang reproducers during clang build |
| crash_report_dir = build_dir / "clang-crashreports" |
| env = { |
| "CLANG_MODULE_CACHE_PATH": "", |
| "CLANG_CRASH_DIAGNOSTICS_DIR": crash_report_dir, |
| } |
| |
| if api.platform.is_linux: |
| env_prefixes = {"PATH": [cipd_dir.joinpath("binutils-gdb", "bin")]} |
| else: |
| env_prefixes = {} |
| |
| def maybe_run_tests(step_name, projects): |
| if host_platform == target_platform: |
| with api.context(env=env, env_prefixes=env_prefixes), resultdb_context( |
| api, build_dir |
| ): |
| api.ninja( |
| step_name, |
| [f"-j{int(api.goma.jobs)}", "-k0"] |
| + [ |
| ("{stage2_}check-" + project).format(**arguments) |
| for project in projects |
| ], |
| build_dir=build_dir, |
| ) |
| |
| # Upload all clang dependencies to cas. |
| install_dirs = api.file.glob_paths( |
| "find dependency install dirs", staging_dir, "*install*" |
| ) |
| if build_lldb: |
| # This directory contains CPython, since it's installed with LLDB. |
| install_dirs.append(lldb_pkg_dir) |
| |
| api.cas_util.upload( |
| staging_dir, |
| install_dirs, |
| step_name="upload dependencies", |
| output_property="deps_isolated", |
| ) |
| |
| with api.step.nest("clang"), api.context(env=env): |
| api.cmake( |
| step_name="configure", |
| src_dir=llvm_dir / "llvm", |
| build_dir=build_dir, |
| extra_args=[option.format(**arguments) for option in options] |
| + [ |
| "-C", |
| llvm_dir.joinpath( |
| "clang", |
| "cmake", |
| "caches", |
| "Fuchsia{_stage2}.cmake".format(**arguments), |
| ), |
| ], |
| ) |
| api.file.read_text( |
| "read CMakeConfigureLog.yaml", |
| build_dir.joinpath("CMakeFiles", "CMakeConfigureLog.yaml"), |
| ) |
| |
| targets = ["{stage2_}toolchain-distribution".format(**arguments)] |
| if build_lldb: |
| targets.append("{stage2_}debugger-distribution".format(**arguments)) |
| |
| try: |
| # Build the full (two-stage) distribution. |
| api.ninja( |
| "build", |
| targets, |
| # This only applies to the first stage, second stage is |
| # invoked by CMake as a subprocess and will use Ninja's |
| # default. |
| ninja_jobs=ninja_jobs, |
| build_dir=build_dir, |
| ) |
| except api.step.StepFailure: |
| upload_crash_reproducer( |
| api, |
| crash_report_dir, |
| props.artifact_gcs_bucket, |
| api.buildbucket_util.id, |
| ) |
| |
| # Extract build traces. |
| upload_build_traces( |
| api, |
| use_ninjatrace, |
| cipd_dir, |
| build_dir, |
| props.artifact_gcs_bucket, |
| api.buildbucket_util.id, |
| ) |
| |
| with api.context(env={"DESTDIR": pkg_dir}): |
| api.ninja( |
| "install", |
| [ |
| "{stage2_}install-toolchain-distribution".format(**arguments), |
| ], |
| build_dir=build_dir, |
| ) |
| |
| if build_lldb: |
| with api.context(env={"DESTDIR": lldb_pkg_dir}): |
| api.ninja( |
| "install", |
| [ |
| "{stage2_}install-debugger-distribution".format( |
| **arguments |
| ), |
| ], |
| build_dir=build_dir, |
| ) |
| if api.platform.is_win: |
| # Copy python.dll to lldb.exe directory due to Windows |
| # DLL search path limitations. |
| dllname = f"{PYTHON_MAJOR_MINOR.replace('.','')}.dll" |
| api.file.copy( |
| "copy python.dll", |
| cpython_install_dir / dllname, |
| lldb_pkg_dir.joinpath("bin", dllname), |
| ) |
| |
| # Upload toolchain to CAS before running clang tests. |
| with api.context(cwd=api.path.start_dir): |
| clang_version = read_clang_version_from_build( |
| api, build_dir, props.do_2stage |
| ) |
| generate_clang_runtime_json( |
| api, |
| pkg_dir, |
| cipd_dir, |
| use_breakpad, |
| clang_version, |
| ) |
| generate_license( |
| api, |
| llvm_dir, |
| pkg_dir, |
| llvm_projects + llvm_runtimes, |
| libxml2_dir_target, |
| should_build_ncurses, |
| should_build_cpython, |
| is_lldb=False, |
| ) |
| cas_digest = api.cas_util.upload(pkg_dir, output_property="isolated") |
| |
| if build_lldb: |
| with api.step.nest("lldb"): |
| generate_license( |
| api, |
| llvm_dir, |
| lldb_pkg_dir, |
| ["clang", "llvm"], |
| libxml2_dir_target, |
| should_build_ncurses, |
| should_build_cpython, |
| is_lldb=True, |
| ) |
| api.cas_util.upload(lldb_pkg_dir, output_property="lldb_isolated") |
| |
| # Run the tests. |
| projects = ["clang", "lld", "llvm", "polly"] |
| # TODO(leonardchan): run host runtime tests for mac. |
| if ( |
| host_triple |
| in [ |
| "x86_64-unknown-linux-gnu", |
| "aarch64-unknown-linux-gnu", |
| "x86_64-pc-windows-msvc", |
| ] |
| and not props.do_2stage |
| ): |
| projects += ["runtimes-" + host_triple] |
| # TODO(fxbug.dev/337333427): Remove this workaround in libcxx tests once sys_info.zdump.pass.cpp is no longer flaky |
| filter_out_regexps = [ |
| "std/time/time.zone/time.zone.timezone/time.zone.members/sys_info.zdump.pass.cpp", |
| ] |
| with api.context( |
| env={ |
| "LIT_FILTER_OUT": "|".join(filter_out_regexps), |
| } |
| ): |
| maybe_run_tests("test", projects) |
| |
| def maybe_upload(name, pkg_dir): |
| if props.upload_package and clang_version: |
| # The published package has the same name for every platform. |
| api.cipd_util.upload_package( |
| f"fuchsia/third_party/{name}/{target_platform}", |
| pkg_dir, |
| search_tag={"git_revision": revision}, |
| repository=props.repository, |
| metadata=[("version", clang_version)], |
| ) |
| |
| # Upload clang package and do triggers if the tests didn't fail. |
| maybe_upload("clang", pkg_dir) |
| |
| # TODO(phosek): move this logic to clang_trigger.py recipe. |
| # Suppress triggering from the RBE-built tools for now. |
| if ( |
| cas_digest |
| and not props.use_rbe |
| and not target_platform |
| in [ |
| "linux-arm64", |
| "windows-amd64", |
| ] |
| ): |
| # Do a full integration build. This will use the just-built toolchain |
| # to build all of Fuchsia to check whether there are any regressions. |
| if props.builders and target_platform in props.builders: |
| api.toolchain.trigger_build( |
| "clang_toolchain", |
| props.repository, |
| revision, |
| cas_digest, |
| builders=props.builders[target_platform], |
| fuchsia_ref="refs/heads/releases/canary", |
| ) |
| if props.tot_builders and target_platform in props.tot_builders: |
| api.toolchain.trigger_build( |
| "clang_toolchain", |
| props.repository, |
| revision, |
| cas_digest, |
| builders=props.tot_builders[target_platform], |
| fuchsia_ref="refs/heads/main", |
| ) |
| |
| # TODO(https://fxbug.dev/42074681): Enable LLDB tests on Windows once python binding is working. |
| if build_lldb: |
| with api.step.nest("lldb"): |
| xfail_tests = [ |
| # http://fxbug.dev/132371 |
| "lldb-api :: commands/expression/import-std-module/array/TestArrayFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/deque-basic/TestDequeFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/queue/TestQueueFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVectorFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectorsFromStdModule.py", |
| # http://fxbug.dev/132374 |
| "lldb-api :: lang/c/global_variables/TestGlobalVariables.py", |
| "lldb-api :: lang/cpp/char8_t/TestCxxChar8_t.py", |
| # http://fxbug.dev/132375 |
| "lldb-api :: functionalities/data-formatter/data-formatter-stl/libcxx/function/TestLibCxxFunction.py", |
| "lldb-api :: lang/cpp/std-function-step-into-callable/TestStdFunctionStepIntoCallable.py", |
| # http://fxbug.dev/132379 |
| "lldb-api :: functionalities/target-new-solib-notifications/TestModuleLoadedNotifys.py", |
| # http://fxbug.dev/132431 |
| "lldb-api :: lang/cpp/stl/TestStdCXXDisassembly.py", |
| ] |
| filter_regexps = [] |
| filter_out_regexps = [ |
| # Fail to build test application |
| "data-formatter-stl/(generic|libstdcpp)", |
| "functionalities/postmortem/FreeBSDKernel/TestFreeBSDKernelVMCore.py", |
| "iohandler/unicode/TestUnicode.py", |
| # TODO(fxbug.dev/331486430): TestConcurrentVFork potentially leave a few process without |
| # cleaning up, causing the test to time out. Disable it for now. |
| "functionalities/fork/concurrent_vfork/TestConcurrentVFork.py", |
| # TODO(https://issues.fuchsia.dev/337293279) Once we can |
| # handle psutil tests, TestDAP_console.py should be removed |
| # from this list |
| "tools/lldb-dap/console/TestDAP_console.py", |
| ] |
| if api.platform.is_linux: |
| xfail_tests.extend( |
| [ |
| # http://fxbug.dev/132374 |
| "lldb-api :: commands/target/basic/TestTargetCommand.py", |
| ] |
| ) |
| elif api.platform.is_mac: |
| # Just run a smoke test. See http://fxbug.dev/319159788. |
| filter_regexps.append("commands/expression") |
| |
| if target_platform == "linux-arm64": |
| xfail_tests.extend( |
| [ |
| # http://fxbug.dev/322406146 |
| "lldb-api :: commands/expression/import-std-module/basic/TestImportStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py", |
| "lldb-api :: commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardListFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/forward_list/TestForwardListFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/iterator/TestIteratorFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentListFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/list/TestListFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/retry-with-std-module/TestRetryWithStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py", |
| "lldb-api :: commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py", |
| "lldb-api :: commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py", |
| ] |
| ) |
| else: |
| xfail_tests.extend( |
| [ |
| # http://fxbug.dev/132425 |
| "lldb-api :: lang/c/calling-conventions/TestCCallingConventions.py", |
| ] |
| ) |
| if should_run_lldb_test: |
| with api.context( |
| env={ |
| "LIT_XFAIL": ";".join(xfail_tests), |
| "LIT_FILTER": "|".join(filter_regexps), |
| "LIT_FILTER_OUT": "|".join(filter_out_regexps), |
| } |
| ): |
| # Separately test lldb and upload it if those tests succeed. |
| # If this step fails, it prevents uploading lldb but not clang. |
| maybe_run_tests("test", ["lldb"]) |
| maybe_upload("lldb", lldb_pkg_dir) |
| |
| |
| def checkout_source(api, git, ref, src_dir, manifest): |
| _, revision = api.git_checkout(git, path=src_dir, revision=ref) |
| git_checkout = manifest.directories[str(src_dir)].git_checkout |
| git_checkout.repo_url = git |
| git_checkout.revision = revision |
| |
| |
| @contextlib.contextmanager |
| def resultdb_context(api, build_dir): |
| try: |
| yield |
| finally: |
| upload_to_resultdb(api, build_dir) |
| |
| |
| def upload_to_resultdb(api, build_dir): |
| if not api.resultdb.enabled: |
| return # pragma: no cover |
| test_data = f"{build_dir.joinpath(RESULTDB_JSON)}\n{build_dir.joinpath('test', RESULTDB_JSON)}\n" |
| # api.path.glob does not handle long path on |
| # windows, use a python3 script to mitigate |
| # this issue. fxbug.dev/98099 |
| results = api.python3( |
| "collect results.json", |
| [ |
| api.resource("find_files.py"), |
| "--dir", |
| build_dir, |
| "--pattern", |
| f"**/{RESULTDB_JSON}", |
| ], |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text(test_data), |
| stdout=api.raw_io.output_text(add_output_log=True), |
| ) |
| results_paths = results.stdout.splitlines() |
| if not results_paths: |
| return # pragma: no cover |
| resultdb_base_variant = { |
| "bucket": api.buildbucket.build.builder.bucket, |
| "builder": api.buildbucket.build.builder.builder, |
| } |
| cmd = ["vpython3", api.resource("resultdb.py")] |
| cmd.extend(f"--json={p}" for p in results_paths) |
| api.step( |
| "resultdb", |
| api.resultdb.wrap(cmd, base_variant=resultdb_base_variant.copy(), include=True), |
| ) |
| |
| |
| def fetch_and_extract_windows_python(api, py_version, install_dir): |
| list_of_installers = [ |
| f"https://www.python.org/ftp/python/{py_version}/amd64/core.msi", |
| f"https://www.python.org/ftp/python/{py_version}/amd64/dev.msi", |
| f"https://www.python.org/ftp/python/{py_version}/amd64/doc.msi", |
| f"https://www.python.org/ftp/python/{py_version}/amd64/exe.msi", |
| f"https://www.python.org/ftp/python/{py_version}/amd64/lib.msi", |
| ] |
| download_dir = api.path.start_dir / "msi" |
| api.file.ensure_directory("create python download dir", download_dir) |
| downloaded_installer = [] |
| with api.step.nest("fetch cpython"): |
| for item in list_of_installers: |
| filename = item.split("/")[-1].strip() |
| file_path = download_dir / filename |
| api.url.get_file(item, file_path, step_name=f"Download {filename}") |
| downloaded_installer.append(file_path) |
| with api.step.nest("extract cpython"): |
| for item in downloaded_installer: |
| filename = str(item) |
| api.step( |
| f"install {filename}", |
| [ |
| "msiexec.exe", |
| "/a", |
| item, |
| f"targetdir={install_dir}", |
| "/quiet", |
| ], |
| ) |
| |
| |
| def build_libxml2( |
| api, |
| options, |
| arguments, |
| destdir, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| dir_suffix, |
| ): |
| # libxml2 requires CMAKE_INSTALL_PREFIX to be set to a valid path so replace |
| # an empty prefix with the destination directory. |
| install_prefix = "-DCMAKE_INSTALL_PREFIX=" |
| options = [ |
| install_prefix + str(destdir) if option == install_prefix else option |
| for option in options |
| ] |
| |
| libxml2_dir = api.path.start_dir.joinpath("libxml2%s" % dir_suffix) |
| src_dir = libxml2_dir / "src" |
| checkout_source(api, LIBXML2_GIT, "refs/tags/v2.9.12", src_dir, manifest) |
| build_dir = libxml2_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| api.cmake.build_with_ninja( |
| src_dir=src_dir, |
| build_dir=build_dir, |
| build_type="Release", |
| extra_args=[option.format(**arguments) for option in options] |
| + [ |
| "-DBUILD_SHARED_LIBS=OFF", |
| "-DLIBXML2_WITH_ICONV=OFF", |
| "-DLIBXML2_WITH_ICU=OFF", |
| "-DLIBXML2_WITH_LZMA=OFF", |
| "-DLIBXML2_WITH_PYTHON=OFF", |
| "-DLIBXML2_WITH_TESTS=OFF", |
| # TODO(phosek): lld only links libxml2 and not zlib, so when |
| # zlib support in libxml2 is enabled, this fails in the case of |
| # static linking. |
| "-DLIBXML2_WITH_ZLIB=OFF", |
| ], |
| ninja_jobs=ninja_jobs, |
| ) |
| return destdir.joinpath("lib", "cmake", "libxml2-2.9.10"), libxml2_dir |
| |
| |
| def build_boringssl(api, options, arguments, destdir, ninja_jobs, cipd_dir, manifest): |
| base_dir = api.path.start_dir / "boringssl" |
| src_dir = base_dir / "src" |
| checkout_source( |
| api, |
| BORINGSSL_GIT, |
| "01d195bd03bfff54dc99c0df0858197c71d35417", |
| src_dir, |
| manifest, |
| ) |
| |
| build_dir = base_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| api.cmake.build_with_ninja( |
| src_dir=src_dir, |
| build_dir=build_dir, |
| install_dir=destdir, |
| build_type="Release", |
| extra_args=[option.format(**arguments) for option in options], |
| ninja_jobs=ninja_jobs, |
| ) |
| return destdir.joinpath("lib", "cmake", "OpenSSL") |
| |
| |
| def build_curl(api, options, arguments, destdir, ninja_jobs, cipd_dir, manifest): |
| base_dir = api.path.start_dir / "curl" |
| src_dir = base_dir / "src" |
| checkout_source(api, CURL_GIT, "refs/tags/curl-7_82_0", src_dir, manifest) |
| build_dir = base_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| api.cmake.build_with_ninja( |
| src_dir=src_dir, |
| build_dir=build_dir, |
| install_dir=destdir, |
| build_type="Release", |
| extra_args=[option.format(**arguments) for option in options] |
| + [ |
| "-DBUILD_SHARED_LIBS=OFF", |
| "-DCURL_USE_OPENSSL=ON", |
| ], |
| ninja_jobs=ninja_jobs, |
| ) |
| return destdir.joinpath("lib", "cmake", "CURL") |
| |
| |
| def build_yasm(api, options, arguments, destdir, ninja_jobs, cipd_dir, manifest): |
| base_dir = api.path.start_dir / "yasm" |
| src_dir = base_dir / "src" |
| checkout_source(api, YASM_GIT, "refs/tags/v1.3.0", src_dir, manifest) |
| |
| build_dir = base_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| api.cmake.build_with_ninja( |
| src_dir=src_dir, |
| build_dir=build_dir, |
| install_dir=destdir, |
| build_type="Release", |
| extra_args=[option.format(**arguments) for option in options], |
| ninja_jobs=ninja_jobs, |
| ) |
| |
| |
| def build_cpp_httplib(api, options, arguments, destdir, ninja_jobs, cipd_dir, manifest): |
| # cpp-httplib requires CMAKE_INSTALL_PREFIX to be set to a valid path so |
| # replace an empty prefix with the destination directory. |
| install_prefix = "-DCMAKE_INSTALL_PREFIX=" |
| options = [ |
| install_prefix + str(destdir) if option == install_prefix else option |
| for option in options |
| ] |
| |
| base_dir = api.path.start_dir / "cpp-httplib" |
| src_dir = base_dir / "src" |
| checkout_source(api, CPP_HTTPLIB_GIT, "refs/tags/v0.12.2", src_dir, manifest) |
| build_dir = base_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| api.cmake( |
| step_name="configure", |
| src_dir=src_dir, |
| build_dir=build_dir, |
| build_type="Release", |
| extra_args=[option.format(**arguments) for option in options] |
| + [ |
| "-DHTTPLIB_USE_BROTLI_IF_AVAILABLE=OFF", |
| # TODO(fxbug.dev/125958): Enable boringssl in cpp-httplib. |
| "-DHTTPLIB_USE_OPENSSL_IF_AVAILABLE=OFF", |
| "-DHTTPLIB_REQUIRE_ZLIB=ON", |
| ], |
| ) |
| # This is a header-only library, so there's no build step. |
| api.ninja( |
| "install", |
| ["install"], |
| build_dir=build_dir, |
| ) |
| return destdir.joinpath("lib", "cmake", "httplib") |
| |
| |
| COMMON_CONFIGURE_OPTIONS = [ |
| "CC={cc}", |
| "CXX={cxx}", |
| "AR={ar}", |
| "LD={ld}", |
| "RANLIB={ranlib}", |
| "NM={nm}", |
| "STRIP={strip}", |
| "OBJCOPY={objcopy}", |
| ] |
| |
| |
| def build_ncurses(api, arguments, destdir, manifest): |
| base_dir = api.path.start_dir / "ncurses" |
| src_dir = base_dir / "src" |
| checkout_source(api, NCURSES_GIT, "refs/tags/v6.4", src_dir, manifest) |
| |
| build_dir = base_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| |
| with api.context(cwd=build_dir): |
| api.step( |
| "configure", |
| [src_dir / "configure"] |
| + [ |
| option.format(**arguments) |
| for option in [ |
| "--prefix=", |
| "--host={target_triple}", |
| "--with-terminfo-dirs=/etc/terminfo:/lib/terminfo:/usr/share/terminfo", |
| "--with-default-terminfo-dir=/usr/share/terminfo", |
| "--disable-stripping", |
| *COMMON_CONFIGURE_OPTIONS, |
| "CPPFLAGS=--sysroot={host_sysroot} --target={target_triple}", |
| "CFLAGS={cflags} -fPIC -O3", |
| "CXXFLAGS={cxxflags} -fPIC -O3", |
| "LDFLAGS=--sysroot={host_sysroot} --target={target_triple} {ldflags}", |
| ] |
| ], |
| ) |
| |
| api.step("build", ["make", f"-j{api.platform.cpu_count}"]) |
| api.step("install", ["make", "install", f"DESTDIR={destdir}"]) |
| |
| |
| def build_libedit(api, arguments, ncurses_install_dir, destdir, manifest): |
| base_dir = api.path.start_dir / "libedit" |
| src_dir = base_dir / "src" |
| checkout_source(api, LIBEDIT_GIT, "refs/tags/v20221030-3.1", src_dir, manifest) |
| |
| build_dir = base_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| |
| with api.context(cwd=build_dir): |
| # Git does preserve timestamps, so the configure script attempts to re-run autotools. |
| # See https://www.gnu.org/software/automake/manual/html_node/CVS.html |
| api.step( |
| "touch generated files", |
| [ |
| "touch", |
| src_dir / "aclocal.m4", |
| src_dir / "configure", |
| src_dir / "config.h.in", |
| *api.file.glob_paths("find makefiles", src_dir, "**/Makefile.in"), |
| ], |
| ) |
| api.step( |
| "configure", |
| [src_dir / "configure"] |
| + [ |
| option.format(**arguments) |
| for option in [ |
| "--prefix=", |
| "--host={target_triple}", |
| "--disable-shared", |
| "--with-pic", |
| *COMMON_CONFIGURE_OPTIONS, |
| # Note: --with-sysroot does not affect compilation tests, |
| # so using it fails on MacOS. |
| "CPPFLAGS=--sysroot={host_sysroot} --target={target_triple}" |
| + f" -I{ncurses_install_dir.joinpath('include')} -I{ncurses_install_dir.joinpath('include', 'ncurses')}", |
| "CFLAGS=-O3 {cflags}", |
| "LDFLAGS=--sysroot={host_sysroot} --target={target_triple} {ldflags}" |
| + f" -L{ncurses_install_dir / 'lib'}", |
| ] |
| ], |
| ) |
| api.step("build", ["make", f"-j{api.platform.cpu_count}"]) |
| api.step("install", ["make", "install", f"DESTDIR={destdir}"]) |
| |
| |
| def build_libffi(api, arguments, destdir, manifest): |
| base_dir = api.path.start_dir / "libffi" |
| src_dir = base_dir / "src" |
| checkout_source(api, LIBFFI_GIT, "refs/tags/v3.4.4", src_dir, manifest) |
| |
| build_dir = base_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| |
| with api.context(cwd=build_dir): |
| api.step( |
| "configure", |
| [src_dir / "configure"] |
| + [ |
| option.format(**arguments) |
| for option in [ |
| "--prefix=", |
| "--host={target_triple}", |
| "--disable-docs", |
| "--disable-shared", |
| *COMMON_CONFIGURE_OPTIONS, |
| "CPPFLAGS=--sysroot={host_sysroot} --target={target_triple}", |
| "CFLAGS=-O3 -fPIC -Wno-error=implicit-function-declaration {cflags}", |
| "LDFLAGS=--sysroot={host_sysroot} --target={target_triple} {ldflags}", |
| ] |
| ], |
| ) |
| api.step("build", ["make", f"-j{api.platform.cpu_count}"]) |
| api.step("install", ["make", "install", f"DESTDIR={destdir}"]) |
| |
| |
| def build_cpython( |
| api, |
| arguments, |
| libedit_install_dir, |
| libffi_install_dir, |
| ncurses_install_dir, |
| zlib_install_dir, |
| destdir, |
| manifest, |
| ): |
| base_dir = api.path.start_dir / "cpython" |
| src_dir = base_dir / "src" |
| checkout_source( |
| api, CPYTHON_GIT, "refs/tags/v" + CPYTHON_VERSION, src_dir, manifest |
| ) |
| |
| # Rpath works differently on Darwin, and Python's Makefile already includes |
| # irrevocable handling of it that precludes a portable installation. Patch |
| # the Makefile to set up a relative dylib search. |
| if api.platform.is_mac: |
| with api.context(cwd=src_dir): |
| api.git.apply_patchfile(api.resource("cpython-darwin.patch")) |
| |
| build_dir = base_dir / "build" |
| api.file.ensure_directory("make build dir", build_dir) |
| |
| # These directories have their include and lib directories added formulaically. |
| dep_dirs = [ |
| libedit_install_dir, |
| libffi_install_dir, |
| ncurses_install_dir, |
| zlib_install_dir, |
| ] |
| |
| include_args = "--sysroot={host_sysroot} " + " ".join( |
| [f"-I{dep_dir / 'include'}" for dep_dir in dep_dirs] |
| + [f"-I{ncurses_install_dir.joinpath('include').joinpath('ncurses')}"] |
| ) |
| |
| cpython_ldflags = "" |
| if not api.platform.is_mac: |
| cpython_ldflags = "-Wl,-rpath,'$$ORIGIN'/../lib " |
| cpython_ldflags += "--sysroot={host_sysroot} " + " ".join( |
| [f"-L{dep_dir / 'lib'}" for dep_dir in dep_dirs] |
| ) |
| |
| with api.context(cwd=build_dir): |
| api.step( |
| "configure", |
| [src_dir / "configure"] |
| + [ |
| option.format(**arguments) |
| for option in [ |
| # While this leaks the destination directory into Python, |
| # Python's build and install process currently requires a |
| # real prefix to work properly. It's possible to fake the |
| # prefix with "/.", but this breaks the later installation |
| # of pip and setuptools. |
| f"--prefix={destdir}", |
| "--enable-shared", |
| "--enable-optimizations", |
| "--with-readline=editline", |
| *COMMON_CONFIGURE_OPTIONS, |
| f"CPPFLAGS={include_args}", |
| # On Darwin CPPFLAGS isn't forwarded to the module build. |
| f"CFLAGS=-O3 {include_args}" + " {cflags}", |
| f"LDFLAGS={cpython_ldflags}" + " {ldflags}", |
| ] |
| ], |
| ) |
| api.step("build", ["make", f"-j{api.platform.cpu_count}"]) |
| api.step("install", ["make", "install"]) |
| |
| |
| # Derive the string "python<MAJOR>.<MINOR>" from the version. |
| PYTHON_MAJOR_MINOR = f'python{".".join(CPYTHON_VERSION.split(".")[:2])}' |
| |
| |
| # Make a "lldb" keychain containing a self-signing key "lldb_codesign". Both |
| # LLDB's server and test binaries must be signed to allow attaching a debugger |
| # in the test suite without GUI interaction. Making a new keychain is somewhat |
| # circuitous compared to importing the key into the system or user keychain, |
| # but both of those require GUI interaction, while this procedure can succeed |
| # headlessly, and it still allows codesign to use the key. |
| def make_lldb_keychain(api): |
| # Builders are not hermetic, so delete the keychain if it already exists. |
| api.step( |
| "delete keychain", |
| ["security", "delete-keychain", "lldb"], |
| ok_ret="any", |
| ) |
| api.step( |
| "create keychain", |
| ["security", "create-keychain", "-p", "lldb_codesign", "lldb"], |
| ) |
| api.step( |
| "add keychain to search list", |
| ["security", "list-keychains", "-s", "lldb"], |
| ) |
| api.step( |
| "remove keychain default settings", |
| ["security", "set-keychain-settings", "lldb"], |
| ) |
| api.step( |
| "import cert", |
| [ |
| "security", |
| "import", |
| api.resource("lldb_codesign.p12"), |
| "-P", |
| "lldb_codesign", |
| "-k", |
| "lldb", |
| "-T", |
| "/usr/bin/codesign", |
| ], |
| ) |
| api.step( |
| "set keychain ACL", |
| [ |
| "security", |
| "set-key-partition-list", |
| "-S", |
| "apple-tool:,apple:", |
| "-s", |
| "-k", |
| "lldb_codesign", |
| "lldb", |
| ], |
| ) |
| |
| |
| # //zircon/public/gn/toolchain/clang.gni:clang_runtime sets the JSON schema. |
| # |
| # This function makes the prototype spec; debug and breakpad info is added by |
| # runtimes.py. |
| def make_runtimes_spec(clang_version): |
| # TODO(fxbug.dev/27110): Ideally this would be done by the cmake build itself. |
| runtimes = [] |
| for arch, has_hwasan in [("aarch64", True), ("riscv64", True), ("x86_64", False)]: |
| for mode_ldflags in [[], ["-static-libstdc++"]]: |
| for mode_sanitizer_cflags, mode_sanitizer_multilibs, mode_runtimes in [ |
| ([], [], []), |
| (["-fsanitize=address"], ["asan"], ["libclang_rt.asan.so"]), |
| ( |
| ["-fsanitize=undefined"], |
| [], |
| ["libclang_rt.ubsan_standalone.so"], |
| ), |
| ] + ( |
| [ |
| ( |
| ["-fsanitize=hwaddress"], |
| ["hwasan"], |
| ["libclang_rt.hwasan.so"], |
| ), |
| ] |
| if has_hwasan |
| else [] |
| ): |
| target_triple = f"{arch}-unknown-fuchsia" |
| target = [target_triple, f"{arch}-fuchsia"] |
| runtime_dir = f"clang/{clang_version}/lib/{target_triple}" |
| stdlib_dir = f"{target_triple}" |
| mode_cflags = mode_sanitizer_cflags |
| mode_multilib = "+".join(mode_sanitizer_multilibs) |
| if mode_multilib: |
| mode_multilib += "/" |
| runtime = [ |
| {"dist": runtime_dir + "/" + soname} for soname in mode_runtimes |
| ] |
| if not mode_ldflags: |
| stdlib = [ |
| { |
| "dist": stdlib_dir + "/" + mode_multilib + soname, |
| "name": soname.split(".")[0], |
| } |
| for soname in [ |
| "libc++.so.2", |
| "libc++abi.so.1", |
| "libunwind.so.1", |
| ] |
| ] |
| runtime.extend(stdlib) |
| |
| runtimes.append( |
| { |
| "target": target, |
| "cflags": mode_cflags, |
| "ldflags": mode_ldflags, |
| "runtime": runtime, |
| } |
| ) |
| |
| return runtimes |
| |
| |
| def extract_trace_filename(api, build_dir, trace): |
| if str(build_dir) not in trace: |
| # trace file should be always in build_dir. |
| # But if it happens, just upload it anyway |
| # instead of throw an exception. |
| return api.path.basename(trace) # pragma: no cover |
| relative_path = str(api.path.relpath(trace, build_dir)) |
| relative_path = relative_path.replace(api.path.sep, "_") |
| return relative_path |
| |
| |
| def upload_crash_reproducer(api, crashreports_dir, gcs_bucket, build_id): |
| with api.step.nest("upload crash reproducer"), api.context(infra_steps=True): |
| temp = api.path.mkdtemp("reproducers") |
| reproducers = api.file.glob_paths( |
| "find reproducers", |
| crashreports_dir, |
| "*.sh", |
| test_data=(crashreports_dir / "foo.sh",), |
| ) |
| for reproducer in reproducers: |
| base = api.path.splitext(api.path.basename(reproducer))[0] |
| files = api.file.glob_paths( |
| f"find {base} files", |
| crashreports_dir, |
| base + ".*", |
| test_data=( |
| crashreports_dir / "foo.sh", |
| crashreports_dir / "foo.cpp", |
| ), |
| ) |
| |
| tgz_basename = f"{base}.tar.gz" |
| tgz_path = temp / tgz_basename |
| package = api.archive.package(crashreports_dir) |
| for f in files: |
| package.with_file(f) |
| package.archive(f"create {tgz_basename}", tgz_path) |
| api.gsutil.upload_namespaced_file( |
| source=tgz_path, |
| bucket=gcs_bucket, |
| subpath=tgz_basename, |
| namespace=build_id, |
| ) |
| |
| |
| def upload_build_traces( |
| api, ninjatrace_available, cipd_dir, build_dir, artifact_gcs_bucket, build_id |
| ): |
| with api.step.nest("ninja log and traces"): |
| test_data = f"{build_dir.joinpath(NINJA_LOG)}\n{build_dir.joinpath('test', NINJA_LOG)}\n" |
| # api.path.glob does not handle long path on |
| # windows, use a python3 script to mitigate |
| # this issue. fxbug.dev/98099 |
| logs = api.python3( |
| "collect .ninja.log", |
| [ |
| api.resource("find_files.py"), |
| "--dir", |
| build_dir, |
| "--pattern", |
| f"**/{NINJA_LOG}", |
| ], |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text(test_data), |
| stdout=api.raw_io.output_text(add_output_log=True), |
| ) |
| log_paths = logs.stdout.splitlines() |
| if not ninjatrace_available: |
| # ninjatrace is not yet available on Linux-arm64 and Windows |
| # platforms. Upload logs for offline analysis. |
| with api.step.nest("upload ninja log"): |
| for log_file in log_paths: |
| filename = extract_trace_filename(api, build_dir, log_file) |
| api.gsutil.upload_namespaced_file( |
| source=log_file, |
| bucket=artifact_gcs_bucket, |
| subpath=filename, |
| namespace=build_id, |
| ) |
| return |
| trace_paths = [] |
| with api.step.nest("generate ninja traces"): |
| for log_file in log_paths: |
| trace = log_file + ".json" |
| try: |
| api.step( |
| "ninjatrace", |
| [ |
| cipd_dir.joinpath("ninjatrace", "ninjatrace"), |
| "-ninjalog", |
| log_file, |
| # TODO(haowei): add compdb |
| "-trace-json", |
| trace, |
| ], |
| ) |
| trace_paths.append(trace) |
| except api.step.StepFailure: # pragma: no cover |
| pass |
| with api.step.nest("upload traces"): |
| for trace in trace_paths: |
| filename = extract_trace_filename(api, build_dir, trace) |
| # Upload may fail if service account don't have access to bucket. |
| # Ignore the error for now since upload failures do not impact the |
| # toolchain build process. |
| try: |
| step = api.gsutil.upload_namespaced_file( |
| source=trace, |
| bucket=artifact_gcs_bucket, |
| subpath=filename, |
| namespace=build_id, |
| ) |
| step.presentation.links["perfetto_ui"] = ( |
| f"https://ui.perfetto.dev/#!?url={api.gsutil.unauthenticated_url(step.presentation.links[filename])}" |
| ) |
| except api.step.StepFailure: # pragma: no cover |
| pass |
| |
| |
| def read_clang_version_from_build(api, build_dir, do_2stage): |
| step_result = api.file.read_text( |
| "Version.inc", |
| build_dir.joinpath( |
| *( |
| (["tools", "clang", "stage2-bins"] if do_2stage else []) |
| + ["tools", "clang", "include", "clang", "Basic", "Version.inc"] |
| ) |
| ), |
| test_data='#define CLANG_VERSION_MAJOR_STRING "8"', |
| ) |
| m = re.search(r'CLANG_VERSION_MAJOR_STRING "([a-zA-Z0-9]+)"', step_result) |
| assert m, "Cannot determine Clang version" |
| return m.group(1) |
| |
| |
| def generate_clang_runtime_json( |
| api, |
| pkg_dir, |
| cipd_dir, |
| use_breakpad, |
| clang_version, |
| ): |
| api.toolchain.strip_runtimes( |
| "generate runtime.json", |
| spec=make_runtimes_spec(clang_version), |
| path=pkg_dir / "lib", |
| build_id_subpath="debug/.build-id", |
| readelf=cipd_dir.joinpath("bin", "llvm-readelf"), |
| objcopy=cipd_dir.joinpath("bin", "llvm-objcopy"), |
| dump_syms=( |
| cipd_dir.joinpath("breakpad", "dump_syms", "dump_syms") |
| if use_breakpad |
| else None |
| ), |
| ) |
| |
| |
| def generate_license( |
| api, |
| llvm_dir, |
| pkg_dir, |
| projects, |
| libxml2_dir, |
| built_ncurses, |
| built_cpython, |
| is_lldb, |
| ): |
| license_args = ["--include"] + [ |
| llvm_dir.joinpath(project, "LICENSE.TXT") for project in projects |
| ] |
| |
| if not is_lldb: |
| license_args.extend( |
| [ |
| "--extract", |
| llvm_dir.joinpath("polly", "lib", "External", "isl", "LICENSE"), |
| "-", |
| ] |
| ) |
| |
| license_args.extend( |
| [ |
| "--extract", |
| llvm_dir.joinpath( |
| "llvm", "lib", "Support", "UnicodeNameToCodepointGenerated.cpp" |
| ), |
| "17-63", |
| "--extract", |
| libxml2_dir.joinpath("src", "Copyright"), |
| "-", |
| "--extract", |
| api.zlib.checkout / "zlib.h", |
| "4-22", |
| "--extract", |
| api.path.start_dir.joinpath("boringssl", "src", "LICENSE"), |
| "-", |
| "--extract", |
| api.path.start_dir.joinpath("curl", "src", "COPYING"), |
| "-", |
| ] |
| ) |
| |
| if is_lldb: |
| if built_ncurses: |
| license_args.extend( |
| [ |
| "--extract", |
| api.path.start_dir.joinpath("ncurses", "src", "COPYING"), |
| "-", |
| "--extract", |
| api.path.start_dir.joinpath("libedit", "src", "COPYING"), |
| "-", |
| ] |
| ) |
| if built_cpython: |
| license_args.extend( |
| [ |
| "--extract", |
| api.path.start_dir.joinpath("libffi", "src", "LICENSE"), |
| "-", |
| "--extract", |
| api.path.start_dir.joinpath("cpython", "src", "LICENSE"), |
| "-", |
| ] |
| ) |
| api.step( |
| "generate license", |
| cmd=["python3", api.resource("generate_license.py")] + license_args, |
| stdout=api.raw_io.output_text(leak_to=pkg_dir / "LICENSE"), |
| ) |
| |
| |
| def GenTests(api): |
| builders = { |
| "linux-amd64": ["fuchsia/linux-x64-builder"], |
| "mac-amd64": ["fuchsia/mac-x64-builder"], |
| } |
| tot_builders = { |
| "linux-amd64": ["fuchsia/linux-x64-builder-tot"], |
| } |
| for os in ("linux", "mac"): |
| yield ( |
| api.test(f"ci_{os}_x64") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name(os) |
| + api.properties( |
| platform=os + "-amd64", |
| enable_assertions=True, |
| enable_backtraces=True, |
| builders=builders, |
| tot_builders=tot_builders, |
| reverts=["b" * 40], |
| ) |
| ) |
| |
| yield ( |
| api.test(f"prod_{os}_x64") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="prod", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name(os) |
| + api.properties( |
| platform=os + "-amd64", |
| builders=builders, |
| tot_builders=tot_builders, |
| do_2stage=True, |
| upload_package=True, |
| ) |
| + api.git.get_remote_branch_head("git ls-remote", "b" * 40) |
| ) |
| |
| yield ( |
| api.test("windows_amd64") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("win") |
| + api.properties( |
| platform="windows-amd64", |
| do_2stage=True, |
| builders=builders, |
| tot_builders=tot_builders, |
| ) |
| ) |
| |
| yield ( |
| api.test("linux_arm64") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="prod", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform("linux", arch="arm", bits=64) |
| + api.properties( |
| platform="linux-arm64", |
| do_2stage=True, |
| builders=builders, |
| tot_builders=tot_builders, |
| upload_package=True, |
| ) |
| ) |
| |
| yield ( |
| api.test("mac_arm64") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="prod", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("mac") |
| + api.platform.bits(64) |
| + api.properties( |
| platform="mac-arm64", |
| do_2stage=True, |
| upload_package=False, |
| builders=builders, |
| tot_builders=tot_builders, |
| ) |
| ) |
| |
| yield ( |
| api.test("lld") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("mac") |
| + api.properties( |
| platform="mac-amd64", |
| enable_lld=True, |
| builders=builders, |
| tot_builders=tot_builders, |
| ) |
| ) |
| |
| yield ( |
| api.test("lto") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("linux") |
| + api.properties( |
| platform="linux-amd64", |
| lto_mode=LTOMode.FULL, |
| builders=builders, |
| tot_builders=tot_builders, |
| ) |
| ) |
| |
| yield ( |
| api.test("thinlto") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("linux") |
| + api.properties( |
| platform="linux-amd64", |
| lto_mode=LTOMode.THIN, |
| builders=builders, |
| tot_builders=tot_builders, |
| ) |
| ) |
| |
| yield ( |
| api.test("rbe") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("linux") |
| + api.properties( |
| platform="linux-amd64", |
| builders=builders, |
| tot_builders=tot_builders, |
| use_rbe=True, |
| ) |
| ) |
| |
| yield ( |
| api.test("clang_reproducer") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("linux") |
| + api.properties( |
| platform="linux-amd64", |
| lto_mode=LTOMode.THIN, |
| builders=builders, |
| tot_builders=tot_builders, |
| ) |
| + api.step_data("clang.build", retcode=1) |
| ) |