| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building Clang toolchain.""" |
| |
| from recipe_engine.recipe_api import Property |
| from recipe_engine.config import Enum |
| from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest |
| |
| from google.protobuf import json_format |
| |
| import re |
| |
| DEPS = [ |
| "fuchsia/cipd_platform", |
| "fuchsia/git", |
| "fuchsia/goma", |
| "fuchsia/macos_sdk", |
| "fuchsia/nullcontext", |
| "fuchsia/status_check", |
| "fuchsia/toolchain", |
| "fuchsia/upload", |
| "fuchsia/windows_sdk", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/isolated", |
| "recipe_engine/path", |
| "recipe_engine/platform", |
| "recipe_engine/properties", |
| "recipe_engine/python", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| TARGET_TO_ARCH = { |
| "x64": "x86_64", |
| "arm64": "aarch64", |
| } |
| TARGETS = TARGET_TO_ARCH.keys() |
| |
| PLATFORM_TO_TRIPLE = { |
| "linux-amd64": "x86_64-linux-gnu", |
| "linux-arm64": "aarch64-linux-gnu", |
| "mac-amd64": "x86_64-apple-darwin", |
| "windows-amd64": "x86_64-pc-windows-msvc", |
| } |
| PLATFORMS = PLATFORM_TO_TRIPLE.keys() |
| |
| # TODO(fxbug.dev/27210): remove this and switch to normalized triples. |
| TRIPLE_TO_NORMALIZED_TRIPLE = { |
| "x86_64-linux-gnu": "x86_64-unknown-linux-gnu", |
| "aarch64-linux-gnu": "aarch64-unknown-linux-gnu", |
| "x86_64-apple-darwin": "x86_64-apple-darwin", |
| "x86_64-pc-windows-msvc": "x86_64-pc-windows-msvc", |
| } |
| |
| LIBXML2_GIT = "https://fuchsia.googlesource.com/third_party/libxml2" |
| ZLIB_GIT = "https://fuchsia.googlesource.com/third_party/zlib" |
| |
| CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com" |
| |
| PROPERTIES = { |
| "repository": Property( |
| kind=str, |
| help="Git repository URL", |
| default="https://llvm.googlesource.com/llvm-project", |
| ), |
| "revision": Property(kind=str, help="Git revision", default="refs/heads/main"), |
| "platform": Property(kind=str, help="CIPD platform for the target", default=None), |
| "do_2stage": Property(kind=bool, help="Do a 2-stage build", default=None), |
| "enable_lto": Property( |
| kind=Enum("full", "thin"), help="Enable link-time optimization", default=None |
| ), |
| "enable_lld": Property(kind=bool, help="Enable LLD linker", default=None), |
| "enable_assertions": Property(kind=bool, help="Enable assertions", default=None), |
| } |
| |
| |
| def slashes(api, path): |
| return path.replace("\\", "/") if api.platform.is_win else path |
| |
| |
| def build_zlib(api, options, arguments, destdir, ninja_jobs, cipd_dir, manifest): |
| zlib_dir = api.path["start_dir"].join("zlib") |
| src_dir = zlib_dir.join("src") |
| revision = api.git.checkout(ZLIB_GIT, src_dir, ref="upstream/master") |
| git_checkout = manifest.directories[str(src_dir)].git_checkout |
| git_checkout.repo_url = ZLIB_GIT |
| git_checkout.revision = revision |
| build_dir = zlib_dir.join("build") |
| api.file.ensure_directory("make build dir", build_dir) |
| with api.context(cwd=build_dir): |
| api.step( |
| "configure", |
| [cipd_dir.join("bin", "cmake")] |
| + [slashes(api, option.format(**arguments)) for option in options] |
| + ["-DAMD64=ON", src_dir], |
| ) |
| api.step( |
| "build", [cipd_dir.join("ninja"), "-j%d" % ninja_jobs,], |
| ) |
| with api.context(env={"DESTDIR": destdir}): |
| api.step( |
| "install", [cipd_dir.join("ninja"), "install",], |
| ) |
| |
| |
| def build_libxml2(api, options, arguments, destdir, ninja_jobs, cipd_dir, manifest): |
| # TODO(leonardchan): The current version of libxml2 will fail during |
| # the configuration step if CMAKE_INSTALL_PREFIX is an empty string. As a |
| # workaround, just explicitly set the prefix to the install directory and do |
| # not set it as an environment variable as we normally do. We should remove |
| # this once this issue is addressed in upstream libxml2. |
| install_prefix = "-DCMAKE_INSTALL_PREFIX=" |
| options.remove(install_prefix) |
| options.append(install_prefix + str(destdir)) |
| |
| libxml2_dir = api.path["start_dir"].join("libxml2") |
| src_dir = libxml2_dir.join("src") |
| revision = api.git.checkout( |
| LIBXML2_GIT, src_dir, ref="f93ca3e140a371b26366f747a408588c631e0fd1" |
| ) |
| git_checkout = manifest.directories[str(src_dir)].git_checkout |
| git_checkout.repo_url = LIBXML2_GIT |
| git_checkout.revision = revision |
| build_dir = libxml2_dir.join("build") |
| api.file.ensure_directory("make build dir", build_dir) |
| with api.context(cwd=build_dir): |
| api.step( |
| "configure", |
| [cipd_dir.join("bin", "cmake")] |
| + [slashes(api, option.format(**arguments)) for option in options] |
| + [ |
| "-DBUILD_SHARED_LIBS=OFF", |
| "-DLIBXML2_WITH_ICU=OFF", |
| "-DLIBXML2_WITH_LZMA=OFF", |
| "-DLIBXML2_WITH_PYTHON=OFF", |
| # TODO(phosek): lld only links libxml2 and not zlib, so when |
| # zlib support in libxml2 is enabled, this fails in the case of |
| # static linking. |
| "-DLIBXML2_WITH_ZLIB=OFF", |
| ] |
| + [src_dir], |
| ) |
| api.step( |
| "build", [cipd_dir.join("ninja"), "-j%d" % ninja_jobs,], |
| ) |
| api.step( |
| "install", [cipd_dir.join("ninja"), "install",], |
| ) |
| |
| |
| # //zircon/public/gn/toolchain/clang.gni:clang_runtime sets the JSON schema. |
| # |
| # This function makes the prototype spec; debug and breakpad info is added by |
| # runtimes.py. |
| def make_runtimes_spec(clang_version): |
| # TODO(fxbug.dev/27110): Ideally this would be done by the cmake build itself. |
| runtimes = [] |
| for arch in TARGET_TO_ARCH.itervalues(): |
| target_triple = "{arch}-unknown-fuchsia".format(arch=arch) |
| target = [target_triple, "{arch}-fuchsia".format(arch=arch)] |
| runtime_dir = "clang/{version}/lib/{target}".format( |
| version=clang_version, target=target_triple |
| ) |
| cxx_lib_dir = "{target}/c++".format(target=target_triple) |
| for mode_ldflags in [[], ["-static-libstdc++"]]: |
| for mode_cxxabi_cflags, mode_cxxabi_multilibs in [ |
| ([], []), |
| (["-fexperimental-relative-c++-abi-vtables"], ["relative-vtables"]), |
| ]: |
| for mode_sanitizer_cflags, mode_sanitizer_multilibs, mode_runtimes in [ |
| ([], [], []), |
| (["-fsanitize=address"], ["asan"], ["libclang_rt.asan.so"]), |
| (["-fsanitize=undefined"], [], ["libclang_rt.ubsan_standalone.so"]), |
| ]: |
| mode_cflags = mode_cxxabi_cflags + mode_sanitizer_cflags |
| mode_multilib = "+".join( |
| mode_cxxabi_multilibs + mode_sanitizer_multilibs |
| ) |
| if mode_multilib: |
| mode_multilib += "/" |
| runtime = [ |
| {"dist": runtime_dir + "/" + soname} for soname in mode_runtimes |
| ] |
| if not mode_ldflags: |
| cxx_lib = [ |
| { |
| "dist": cxx_lib_dir + "/" + mode_multilib + soname, |
| "name": soname.split(".")[0], |
| } |
| for soname in [ |
| "libc++.so.2", |
| "libc++abi.so.1", |
| "libunwind.so.1", |
| ] |
| ] |
| runtime.extend(cxx_lib) |
| |
| runtimes.append( |
| { |
| "target": target, |
| "cflags": mode_cflags, |
| "ldflags": mode_ldflags, |
| "runtime": runtime, |
| } |
| ) |
| |
| return runtimes |
| |
| |
| def RunSteps( |
| api, |
| repository, |
| revision, |
| platform, |
| do_2stage, |
| enable_lto, |
| enable_lld, |
| enable_assertions, |
| ): |
| use_goma = ( |
| not api.platform.arch == "arm" and api.platform.bits == 64 |
| ) and not api.platform.is_win |
| if use_goma: |
| api.goma.ensure() |
| ninja_jobs = api.goma.jobs |
| goma_context = api.goma.build_with_goma |
| else: |
| ninja_jobs = api.platform.cpu_count |
| goma_context = api.nullcontext |
| |
| # TODO: builders would ideally set this explicitly |
| if do_2stage is None: |
| do_2stage = api.buildbucket.builder_id.bucket == "prod" |
| if enable_lto is None: |
| if not do_2stage or api.platform.name == "win": |
| enable_lto = False |
| if enable_assertions is None: |
| if not do_2stage: |
| enable_assertions = True |
| |
| gitiles_commit = api.buildbucket.build_input.gitiles_commit |
| if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id: |
| repository = "https://%s/%s" % (gitiles_commit.host, gitiles_commit.project) |
| revision = gitiles_commit.id |
| |
| host_platform = api.cipd_platform.name |
| target_platform = platform or host_platform |
| |
| use_breakpad = host_platform == "linux-amd64" |
| |
| manifest = Manifest() |
| |
| with api.step.nest("ensure_packages"): |
| with api.context(infra_steps=True): |
| cipd_dir = api.path["start_dir"].join("cipd") |
| pkgs = api.cipd.EnsureFile() |
| # We don't have SDK for linux-arm64 or win, but we only need sysroot. |
| if ( |
| api.platform.arch == "arm" and api.platform.bits == 64 |
| ) or api.platform.is_win: |
| pkgs.add_package("fuchsia/sdk/core/linux-amd64", "latest", "sdk") |
| else: |
| pkgs.add_package("fuchsia/sdk/core/${platform}", "latest", "sdk") |
| if api.platform.name == "win": |
| pkgs.add_package("infra/cmake/${platform}", "version:3.16.1") |
| pkgs.add_package("infra/ninja/${platform}", "version:1.9.0") |
| # TODO(phosek): Switch to CIPD package once we have it. |
| api.isolated.download( |
| "msys", |
| isolated_hash="7b23533856ac8a1a2fec614e71df22cc065c7448", |
| output_dir=cipd_dir.join("msys"), |
| ) |
| api.isolated.download( |
| "clang", |
| isolated_hash="fd47a7ec0e608bfc7d08714fdb3ef48585cec60d", |
| output_dir=cipd_dir, |
| ) |
| else: |
| pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration") |
| pkgs.add_package( |
| "fuchsia/third_party/cmake/${platform}", |
| "git_revision:fab301bb9d6d7d1c92db077fcd4789c0eb03203f", |
| ) |
| pkgs.add_package( |
| "fuchsia/third_party/ninja/${platform}", |
| "git_revision:0ccc7886fd4694ae1372d29b4954e2dd3be118be", |
| ) |
| pkgs.add_package( |
| "fuchsia/third_party/sysroot/linux", |
| "tp7-Zyo4pv2SVEoK_eaU6yuKmyxJWcR54vtJKTWpTIYC", |
| "linux", |
| ) |
| if use_breakpad: |
| pkgs.add_package( |
| "fuchsia/tools/breakpad/${platform}", "latest", "breakpad" |
| ) |
| ensured = api.cipd.ensure(cipd_dir, pkgs) |
| for subdir, pins in ensured.iteritems(): |
| directory = manifest.directories[str(cipd_dir.join(subdir))] |
| directory.cipd_server_host = CIPD_SERVER_HOST |
| for pin in pins: |
| directory.cipd_package[pin.package].instance_id = pin.instance_id |
| |
| staging_dir = api.path["start_dir"].join("staging") |
| pkg_dir = staging_dir.join("llvm_install") |
| api.file.ensure_directory("create pkg dir", pkg_dir) |
| |
| with api.context(infra_steps=True): |
| llvm_dir = api.path["start_dir"].join("llvm-project") |
| revision = api.git.checkout( |
| repository, |
| path=llvm_dir, |
| ref=revision, |
| step_test_data=lambda: api.raw_io.test_api.stream_output(revision), |
| ) |
| git_checkout = manifest.directories[str(llvm_dir)].git_checkout |
| git_checkout.repo_url = repository |
| git_checkout.revision = revision |
| |
| target_triple = PLATFORM_TO_TRIPLE[target_platform] |
| host_triple = PLATFORM_TO_TRIPLE[host_platform] |
| |
| with api.macos_sdk(kind="ios"), api.windows_sdk(), goma_context(): |
| if api.platform.name == "linux": |
| target_sysroot = host_sysroot = cipd_dir.join("linux") |
| elif api.platform.name == "mac": |
| # TODO(fxbug.dev/3043): Eventually use our own hermetic sysroot as for Linux. |
| step_result = api.step( |
| "xcrun", |
| ["xcrun", "--sdk", "macosx", "--show-sdk-path"], |
| stdout=api.raw_io.output(name="sdk-path", add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output( |
| "/some/xcode/path" |
| ), |
| ) |
| target_sysroot = host_sysroot = step_result.stdout.strip() |
| elif api.platform.name == "win": |
| target_sysroot = host_sysroot = api.windows_sdk.sdk_dir |
| else: # pragma: no cover |
| assert False, "unsupported platform" |
| |
| arguments = { |
| "target_triple": TRIPLE_TO_NORMALIZED_TRIPLE[target_triple], |
| "host_triple": host_triple, |
| "target_sysroot": target_sysroot, |
| "host_sysroot": host_sysroot, |
| "linux_sysroot": cipd_dir.join("linux"), |
| "fuchsia_sdk": cipd_dir.join("sdk"), |
| } |
| |
| if api.platform.is_win: |
| arguments.update( |
| { |
| "cc": cipd_dir.join("bin", "clang-cl.exe"), |
| "cxx": cipd_dir.join("bin", "clang-cl.exe"), |
| "ar": cipd_dir.join("bin", "llvm-ar.exe"), |
| "ld": cipd_dir.join("bin", "lld-link.exe"), |
| "mt": cipd_dir.join("bin", "llvm-mt.exe"), |
| "nm": cipd_dir.join("bin", "llvm-nm.exe"), |
| "objcopy": cipd_dir.join("bin", "llvm-objcopy.exe"), |
| "objdump": cipd_dir.join("bin", "llvm-objdump.exe"), |
| "ranlib": cipd_dir.join("bin", "llvm-ranlib.exe"), |
| "rc": cipd_dir.join("bin", "llvm-rc.exe"), |
| "readelf": cipd_dir.join("bin", "llvm-readelf.exe"), |
| "strip": cipd_dir.join("bin", "llvm-strip.exe"), |
| "ninja": cipd_dir.join("ninja.exe"), |
| } |
| ) |
| else: |
| arguments.update( |
| { |
| "cc": cipd_dir.join("bin", "clang"), |
| "cxx": cipd_dir.join("bin", "clang++"), |
| "ar": cipd_dir.join("bin", "llvm-ar"), |
| "ld": cipd_dir.join("bin", "ld.lld"), |
| "lipo": cipd_dir.join("bin", "llvm-lipo"), |
| "nm": cipd_dir.join("bin", "llvm-nm"), |
| "objcopy": cipd_dir.join("bin", "llvm-objcopy"), |
| "objdump": cipd_dir.join("bin", "llvm-objdump"), |
| "ranlib": cipd_dir.join("bin", "llvm-ranlib"), |
| "readelf": cipd_dir.join("bin", "llvm-readelf"), |
| "strip": cipd_dir.join("bin", "llvm-strip"), |
| "ninja": cipd_dir.join("ninja"), |
| } |
| ) |
| if use_goma: |
| arguments.update( |
| {"gomacc": api.goma.goma_dir.join("gomacc"),} |
| ) |
| |
| options = [ |
| "-GNinja", |
| "-DCMAKE_MAKE_PROGRAM={ninja}", |
| "-DCMAKE_INSTALL_PREFIX=", |
| "-DCMAKE_C_COMPILER={cc}", |
| "-DCMAKE_CXX_COMPILER={cxx}", |
| "-DCMAKE_ASM_COMPILER={cc}", |
| ] |
| if use_goma: |
| options.extend( |
| [ |
| "-DCMAKE_C_COMPILER_LAUNCHER={gomacc}", |
| "-DCMAKE_CXX_COMPILER_LAUNCHER={gomacc}", |
| "-DCMAKE_ASM_COMPILER_LAUNCHER={gomacc}", |
| ] |
| ) |
| |
| options.extend( |
| { |
| "linux": [ |
| "-DCMAKE_AR={ar}", |
| "-DCMAKE_LINKER={ld}", |
| "-DCMAKE_NM={nm}", |
| "-DCMAKE_OBJCOPY={objcopy}", |
| "-DCMAKE_OBJDUMP={objdump}", |
| "-DCMAKE_RANLIB={ranlib}", |
| "-DCMAKE_READELF={readelf}", |
| "-DCMAKE_STRIP={strip}", |
| ], |
| "mac": ["-DCMAKE_LIPO={lipo}",], |
| "win": [ |
| "-DCMAKE_AR={ar}", |
| "-DCMAKE_LINKER={ld}", |
| "-DCMAKE_NM={nm}", |
| "-DCMAKE_OBJCOPY={objcopy}", |
| "-DCMAKE_OBJDUMP={objdump}", |
| "-DCMAKE_RANLIB={ranlib}", |
| "-DCMAKE_READELF={readelf}", |
| "-DCMAKE_STRIP={strip}", |
| # TODO(phosek): reenable once we update the host toolchain |
| # "-DCMAKE_RC_COMPILER={rc}", |
| # "-DCMAKE_MT={mt}", |
| ], |
| }[api.platform.name] |
| ) |
| |
| options.extend(["-DCMAKE_SYSROOT={target_sysroot}"]) |
| |
| if not api.platform.is_win: |
| # TODO(phosek): Build these even for Windows. |
| with api.step.nest("zlib"): |
| zlib_install_dir = staging_dir.join("zlib_install") |
| api.file.ensure_directory("create zlib_install_dir", zlib_install_dir) |
| build_zlib( |
| api, |
| options |
| + [ |
| "-DCMAKE_%s_FLAGS=-O3 -fPIC --target=%s" % (lang, target_triple) |
| for lang in ["C", "CXX"] |
| ], |
| arguments, |
| zlib_install_dir, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| ) |
| with api.step.nest("libxml2"): |
| libxml2_install_dir = staging_dir.join("libxml2_install") |
| api.file.ensure_directory( |
| "create libxml2_install_dir", libxml2_install_dir |
| ) |
| build_libxml2( |
| api, |
| options |
| + [ |
| "-DCMAKE_%s_FLAGS=-O3 -fPIC --target=%s" % (lang, target_triple) |
| for lang in ["C", "CXX"] |
| ], |
| arguments, |
| libxml2_install_dir, |
| ninja_jobs, |
| cipd_dir, |
| manifest, |
| ) |
| |
| json_manifest = json_format.MessageToJson( |
| manifest, preserving_proto_field_name=True |
| ) |
| api.file.write_text( |
| "source manifest", pkg_dir.join("source_manifest.json"), json_manifest |
| ) |
| |
| # build clang+llvm |
| build_dir = staging_dir.join("llvm_build") |
| api.file.ensure_directory("create llvm build dir", build_dir) |
| |
| arguments.update( |
| { |
| "BOOTSTRAP_": "BOOTSTRAP_", |
| "STAGE2_": "STAGE2_", |
| "stage2_": "stage2-", |
| "_stage2": "", |
| } |
| if do_2stage |
| else {"BOOTSTRAP_": "", "STAGE2_": "", "stage2_": "", "_stage2": "-stage2",} |
| ) |
| |
| llvm_projects = ["clang", "clang-tools-extra", "lld", "llvm", "polly"] |
| llvm_runtimes = ["compiler-rt", "libcxx", "libcxxabi", "libunwind"] |
| |
| if not api.platform.is_win: |
| options.extend( |
| [ |
| "-D{BOOTSTRAP_}ZLIB_INCLUDE_DIR=%s" |
| % zlib_install_dir.join("include"), |
| "-D{BOOTSTRAP_}ZLIB_LIBRARY=%s" |
| % zlib_install_dir.join("lib", "libz.a"), |
| "-D{BOOTSTRAP_}LIBXML2_INCLUDE_DIR=%s" |
| % libxml2_install_dir.join("include", "libxml2"), |
| "-D{BOOTSTRAP_}LIBXML2_LIBRARY=%s" |
| % libxml2_install_dir.join("lib", "libxml2.a"), |
| ] |
| ) |
| if do_2stage: |
| options.extend( |
| [ |
| "-DZLIB_INCLUDE_DIR=%s" % zlib_install_dir.join("include"), |
| "-DZLIB_LIBRARY=%s" % zlib_install_dir.join("lib", "libz.a"), |
| "-DLIBXML2_INCLUDE_DIR=%s" |
| % libxml2_install_dir.join("include", "libxml2"), |
| "-DLIBXML2_LIBRARY=%s" |
| % libxml2_install_dir.join("lib", "libxml2.a"), |
| ] |
| ) |
| |
| if api.platform.name == "linux": |
| if do_2stage: |
| options.extend( |
| ["-D{BOOTSTRAP_}CMAKE_SYSROOT={target_sysroot}"] |
| + [ |
| # BOOTSTRAP_ prefixed flags are passed to the second stage compiler. |
| "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-static-libstdc++" % mode |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| + [ |
| # Unprefixed flags are used by the first stage compiler. |
| "-DCMAKE_%s_LINKER_FLAGS=-static-libstdc++" % mode |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| else: |
| options.extend( |
| [ |
| # BOOTSTRAP_ prefixed flags are passed to the second stage compiler. |
| "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-static-libstdc++" % mode |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| if host_triple != target_triple: # pragma: no cover |
| options.extend( |
| [ |
| "-D{BOOTSTRAP_}CMAKE_SYSTEM_NAME=Linux", |
| "-D{BOOTSTRAP_}CMAKE_C_COMPILER_TARGET={target_triple}", |
| "-D{BOOTSTRAP_}CMAKE_CXX_COMPILER_TARGET={target_triple}", |
| "-D{BOOTSTRAP_}LLVM_DEFAULT_TARGET_TRIPLE={target_triple}", |
| ] |
| ) |
| elif api.platform.name == "mac": |
| if do_2stage: |
| options.extend( |
| ["-D{BOOTSTRAP_}CMAKE_SYSROOT={target_sysroot}"] |
| + [ |
| # BOOTSTRAP_ prefixed flags are passed to the second stage compiler. |
| "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-nostdlib++ %s" |
| % (mode, build_dir.join("lib", "libc++.a")) |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| + [ |
| # Unprefixed flags are used by the first stage compiler. |
| "-DCMAKE_%s_LINKER_FLAGS=-nostdlib++ %s" |
| % (mode, cipd_dir.join("lib", "libc++.a")) |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| else: |
| options.extend( |
| [ |
| # BOOTSTRAP_ prefixed flags are passed to the second stage compiler. |
| "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-nostdlib++ %s" |
| % (mode, cipd_dir.join("lib", "libc++.a")) |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| |
| # STAGE2_ prefixed flags are passed to the second stage by the first stage build. |
| |
| if enable_lto is not None: |
| options.extend(["-D{STAGE2_}LLVM_ENABLE_LTO=%s" % str(enable_lto).title()]) |
| |
| if enable_lld is not None: |
| options.extend(["-D{STAGE2_}LLVM_ENABLE_LLD=%s" % str(enable_lld).title()]) |
| |
| if enable_assertions: |
| options.extend( |
| [ |
| "-D{STAGE2_}LLVM_ENABLE_ASSERTIONS=%s" |
| % str(enable_assertions).title() |
| ] |
| ) |
| |
| options.extend( |
| [ |
| "-D{STAGE2_}LINUX_aarch64-unknown-linux-gnu_SYSROOT={linux_sysroot}", |
| "-D{STAGE2_}LINUX_armv7-unknown-linux-gnueabihf_SYSROOT={linux_sysroot}", |
| "-D{STAGE2_}LINUX_i386-unknown-linux-gnu_SYSROOT={linux_sysroot}", |
| "-D{STAGE2_}LINUX_x86_64-unknown-linux-gnu_SYSROOT={linux_sysroot}", |
| "-D{STAGE2_}FUCHSIA_SDK={fuchsia_sdk}", |
| ] |
| ) |
| |
| if api.platform.is_win: |
| env_prefixes = {"PATH": [cipd_dir.join("msys")]} |
| else: |
| env_prefixes = {} |
| |
| with api.step.nest("clang"), api.context( |
| cwd=build_dir, env_prefixes=env_prefixes |
| ): |
| api.step( |
| "configure", |
| [cipd_dir.join("bin", "cmake")] |
| + [slashes(api, option.format(**arguments)) for option in options] |
| + [ |
| "-C", |
| llvm_dir.join( |
| "clang", |
| "cmake", |
| "caches", |
| "Fuchsia{_stage2}.cmake".format(**arguments), |
| ), |
| llvm_dir.join("llvm"), |
| ], |
| ) |
| # Build the full (two-stage) distribution. |
| api.step( |
| "build", |
| [ |
| cipd_dir.join("ninja"), |
| # This only applies to the first stage, second stage is invoked by |
| # CMake as a subprocess and will use Ninja's default. |
| "-j%d" % ninja_jobs, |
| "{stage2_}distribution".format(**arguments), |
| ], |
| ) |
| # Run the tests. |
| projects = ["clang", "lld", "llvm", "polly"] |
| # TODO(phosek): run runtime tests |
| # + [(runtime + '-{target_triple}').format(**arguments) |
| # for runtime in ['unwind', 'cxxabi', 'cxx']] |
| api.step( |
| "test", |
| [cipd_dir.join("ninja"), "-j%d" % api.goma.jobs] |
| + [ |
| ("{stage2_}check-" + project).format(**arguments) |
| for project in projects |
| ], |
| ) |
| with api.context(env={"DESTDIR": pkg_dir}): |
| api.step( |
| "install", |
| [ |
| cipd_dir.join("ninja"), |
| "{stage2_}install-distribution".format(**arguments), |
| ], |
| ) |
| |
| if api.platform.is_mac: |
| # build libunwind, libcxxabi and libcxx for Apple Silicon |
| runtimes_build_dir = staging_dir.join("runtimes_build") |
| with api.step.nest("runtimes"): |
| runtimes_options = [ |
| "-GNinja", |
| "-DCMAKE_MAKE_PROGRAM={ninja}", |
| "-DCMAKE_INSTALL_PREFIX=", |
| "-DCMAKE_ASM_COMPILER=%s" % pkg_dir.join("bin", "clang"), |
| "-DCMAKE_ASM_COMPILER_TARGET=arm64-apple-darwin", |
| "-DCMAKE_C_COMPILER=%s" % pkg_dir.join("bin", "clang"), |
| "-DCMAKE_C_COMPILER_TARGET=arm64-apple-darwin", |
| "-DCMAKE_CXX_COMPILER=%s" % pkg_dir.join("bin", "clang++"), |
| "-DCMAKE_CXX_COMPILER_TARGET=arm64-apple-darwin", |
| "-DCMAKE_APPLE_SILICON_PROCESSOR=arm64", |
| "-DCMAKE_SYSROOT={target_sysroot}", |
| "-DLLVM_ENABLE_RUNTIMES=libunwind;libcxxabi;libcxx", |
| "-DLIBUNWIND_ENABLE_SHARED=OFF", |
| "-DLIBUNWIND_INSTALL_LIBRARY=OFF", |
| "-DLIBCXXABI_ENABLE_SHARED=OFF", |
| "-DLIBCXXABI_INSTALL_LIBRARY=OFF", |
| "-DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON", |
| "-DLIBCXXABI_USE_LLVM_UNWINDER=ON", |
| "-DLIBCXX_ENABLE_SHARED=OFF", |
| "-DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON", |
| "-DLIBCXX_ABI_VERSION=2", |
| ] |
| api.step( |
| "configure", |
| [cipd_dir.join("bin", "cmake")] |
| + ["-B", runtimes_build_dir, "-S", llvm_dir.join("runtimes")] |
| + [option.format(**arguments) for option in runtimes_options], |
| ) |
| api.step( |
| "build", [cipd_dir.join("ninja"), "-C", runtimes_build_dir,], |
| ) |
| for lib in ["libc++.a", "libc++experimental.a"]: |
| api.step( |
| "lipo %s" % lib, |
| [ |
| pkg_dir.join("bin", "llvm-lipo"), |
| "-create", |
| pkg_dir.join("lib", lib), |
| runtimes_build_dir.join("lib", lib), |
| "-output", |
| pkg_dir.join("lib", lib), |
| ], |
| ) |
| |
| step_result = api.file.read_text( |
| "Version.inc", |
| build_dir.join( |
| *( |
| (["tools", "clang", "stage2-bins"] if do_2stage else []) |
| + ["tools", "clang", "include", "clang", "Basic", "Version.inc"] |
| ) |
| ), |
| test_data='#define CLANG_VERSION_STRING "8.0.0"', |
| ) |
| m = re.search(r'CLANG_VERSION_STRING "([a-zA-Z0-9.-]+)"', step_result) |
| assert m, "Cannot determine Clang version" |
| clang_version = m.group(1) |
| |
| api.toolchain.strip_runtimes( |
| "generate runtime.json", |
| spec=make_runtimes_spec(clang_version), |
| path=pkg_dir.join("lib"), |
| build_id_subpath="debug/.build-id", |
| readelf=cipd_dir.join("bin", "llvm-readelf"), |
| objcopy=cipd_dir.join("bin", "llvm-objcopy"), |
| dump_syms=( |
| cipd_dir.join("breakpad", "dump_syms", "dump_syms") |
| if use_breakpad |
| else None |
| ), |
| ) |
| |
| api.python( |
| "generate license", |
| api.resource("generate_license.py"), |
| args=["--include"] |
| + [ |
| llvm_dir.join(project, "LICENSE.TXT") |
| for project in llvm_projects + llvm_runtimes |
| ] |
| + ( |
| [ |
| "--extract", |
| llvm_dir.join("polly", "lib", "External", "isl", "LICENSE"), |
| "-", |
| "--extract", |
| api.path["start_dir"].join("libxml2", "src", "Copyright"), |
| "-", |
| "--extract", |
| api.path["start_dir"].join("zlib", "src", "zlib.h"), |
| "4-22", |
| ] |
| if not api.platform.is_win |
| else [] |
| ), |
| stdout=api.raw_io.output(leak_to=pkg_dir.join("LICENSE")), |
| ) |
| |
| isolated = api.upload.upload_isolated(pkg_dir) |
| |
| if api.buildbucket.builder_id.bucket == "prod": |
| # The published package has the same name for every platform. |
| api.upload.cipd_package( |
| "fuchsia/third_party/clang/%s" % target_platform, |
| pkg_dir, |
| [api.upload.DirectoryPath(pkg_dir)], |
| {"git_revision": revision}, |
| repository=repository, |
| extra_tags={"version": clang_version}, |
| ) |
| |
| # TODO(phosek): move this logic to clang_trigger.py recipe. |
| if not target_platform in ["linux-arm64", "windows-amd64"]: |
| # Do a full integration build. This will use the just-built toolchain |
| # to build all of Fuchsia to check whether there are any regressions. |
| api.toolchain.trigger_build( |
| "clang_toolchain", |
| repository, |
| revision, |
| isolated, |
| builders={ |
| "linux-amd64": ( |
| "clang_toolchain.fuchsia-arm64-debug", |
| "clang_toolchain.fuchsia-arm64-release", |
| "clang_toolchain.fuchsia-x64-debug", |
| "clang_toolchain.fuchsia-x64-release", |
| ), |
| "mac-amd64": ("clang_toolchain.fuchsia-host-mac",), |
| }[target_platform], |
| ) |
| |
| |
| def GenTests(api): |
| for os in ("linux", "mac"): |
| yield ( |
| api.status_check.test("ci_%s_x64" % os) |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name(os) |
| + api.properties(platform=os + "-amd64") |
| ) |
| |
| yield ( |
| api.status_check.test("prod_%s_x64" % os) |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="prod", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name(os) |
| + api.properties(platform=os + "-amd64") |
| + api.git.get_remote_branch_head("git ls-remote", "b" * 40) |
| ) |
| |
| yield ( |
| api.status_check.test("windows_amd64") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("win") |
| + api.properties(platform="windows-amd64") |
| ) |
| |
| yield ( |
| api.status_check.test("linux_arm64") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("linux") |
| + api.platform.arch("arm") |
| + api.platform.bits(64) |
| + api.properties(platform="linux-arm64") |
| ) |
| |
| yield ( |
| api.status_check.test("mac_lld") |
| + api.buildbucket.ci_build( |
| project="fuchsia", |
| bucket="ci", |
| git_repo="https://fuchsia.googlesource.com/third_party/llvm-project", |
| revision="a" * 40, |
| ) |
| + api.platform.name("mac") |
| + api.properties(platform="mac-amd64", enable_lld=True) |
| ) |
