| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building LLVM.""" |
| |
| import contextlib |
| import re |
| |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/cas_util", |
| "fuchsia/cipd_util", |
| "fuchsia/git_checkout", |
| "fuchsia/goma", |
| "fuchsia/macos_sdk", |
| "fuchsia/status_check", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/platform", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| ARCH_TO_TARGET = { |
| "x86_64": "x64", |
| "aarch64": "arm64", |
| } |
| |
| PLATFORM_TO_TRIPLE = { |
| "linux-amd64": ["x86_64-linux-gnu"], |
| "linux-arm64": ["aarch64-linux-gnu"], |
| "mac-amd64": ["x86_64-apple-darwin"], |
| "fuchsia": ["x86_64-fuchsia", "aarch64-fuchsia"], |
| } |
| PLATFORMS = PLATFORM_TO_TRIPLE.keys() |
| |
| LLVM_PROJECT_GIT = "https://llvm.googlesource.com/llvm-project" |
| |
| # The components to build & distribute. This contains all the dependencies of zxdb, i.e. |
| # LLVMAArch64Disassembler, LLVMX86Disassembler, LLVMDebugInfoDWARF and their recursive dependencies. |
| DEFAULT_COMPONENTS = [ |
| "llvm-headers", |
| "LLVMAArch64Desc", |
| "LLVMAArch64Disassembler", |
| "LLVMAArch64Info", |
| "LLVMAArch64Utils", |
| "LLVMAsmParser", |
| "LLVMBinaryFormat", |
| "LLVMBitReader", |
| "LLVMBitstreamReader", |
| "LLVMCore", |
| "LLVMDebugInfoCodeView", |
| "LLVMDebugInfoMSF", |
| "LLVMDebugInfoDWARF", |
| "LLVMDebugInfoPDB", |
| "LLVMDemangle", |
| "LLVMIRReader", |
| "LLVMMC", |
| "LLVMMCDisassembler", |
| "LLVMMCParser", |
| "LLVMObject", |
| "LLVMProfileData", |
| "LLVMRemarks", |
| "LLVMSupport", |
| "LLVMSymbolize", |
| "LLVMTextAPI", |
| "LLVMX86Desc", |
| "LLVMX86Disassembler", |
| "LLVMX86Info", |
| ] |
| |
| PROPERTIES = { |
| "repository": Property( |
| kind=str, help="Git repository URL", default=LLVM_PROJECT_GIT |
| ), |
| "revision": Property(kind=str, help="Revision", default=None), |
| "platform": Property(kind=str, help="CIPD platform for the target", default=None), |
| "version": Property( |
| kind=str, |
| help="CIPD version of the toolchain package", |
| default="integration", |
| ), |
| "components": Property( |
| kind=list, |
| help="LLVM components to distribute", |
| default=DEFAULT_COMPONENTS, |
| ), |
| } |
| |
| |
| def RunSteps(api, repository, revision, platform, version, components): |
| use_goma = ( |
| not api.platform.arch == "arm" and api.platform.bits == 64 |
| ) and not api.platform.is_win |
| if use_goma: |
| api.goma.ensure() |
| ninja_jobs = api.goma.jobs |
| goma_context = api.goma.build_with_goma() |
| else: |
| ninja_jobs = api.platform.cpu_count |
| goma_context = contextlib.nullcontext() |
| |
| host_platform = api.cipd_util.platform_name |
| platform = platform or host_platform |
| |
| with api.step.nest("ensure_packages"): |
| with api.context(infra_steps=True): |
| cipd_dir = api.path["start_dir"].join("cipd") |
| # TODO: deduplicate this and the clang toolchain recipe |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package("infra/3pp/tools/cmake/${platform}", "version:3.13.5") |
| pkgs.add_package("infra/3pp/tools/ninja/${platform}", "version:1.9.0") |
| pkgs.add_package("fuchsia/third_party/clang/${platform}", version) |
| if platform.startswith("linux"): |
| pkgs.add_package( |
| "fuchsia/third_party/sysroot/linux", |
| "tp7-Zyo4pv2SVEoK_eaU6yuKmyxJWcR54vtJKTWpTIYC", |
| "sysroot", |
| ) |
| if platform.startswith("fuchsia"): |
| pkgs.add_package("fuchsia/sdk/core/${platform}", "latest", "sdk") |
| api.cipd.ensure(cipd_dir, pkgs) |
| |
| staging_dir = api.path["start_dir"].join("staging") |
| pkg_dir = staging_dir.join("root") |
| api.file.ensure_directory("create pkg root dir", pkg_dir) |
| |
| llvm_dir, revision = api.git_checkout(repository) |
| |
| # build llvm |
| with api.macos_sdk(), goma_context: |
| targets = PLATFORM_TO_TRIPLE[platform] |
| for triple in targets: |
| build_dir = staging_dir.join("llvm_%s_build_dir" % triple.replace("-", "_")) |
| api.file.ensure_directory("create %s llvm build dir" % triple, build_dir) |
| |
| if len(targets) > 1: |
| install_dir = staging_dir.join( |
| "llvm_%s_install_dir" % triple.replace("-", "_") |
| ) |
| api.file.ensure_directory( |
| "create %s llvm install dir" % triple, install_dir |
| ) |
| else: |
| install_dir = pkg_dir |
| |
| target = ARCH_TO_TARGET[triple.split("-")[0]] |
| if platform == "fuchsia": |
| sysroot = cipd_dir.join("sdk", "arch", target, "sysroot") |
| elif platform.startswith("linux"): |
| sysroot = cipd_dir.join("sysroot") |
| elif platform.startswith("mac"): |
| # TODO(fxbug.dev/3043): Eventually use our own hermetic sysroot. |
| step_result = api.step( |
| "xcrun", |
| ["xcrun", "--sdk", "macosx", "--show-sdk-path"], |
| stdout=api.raw_io.output_text(name="sdk-path", add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "/some/xcode/path" |
| ), |
| ) |
| sysroot = step_result.stdout.strip() |
| |
| if not platform.startswith("mac"): |
| extra_options = [ |
| "-DCMAKE_AR=%s" % cipd_dir.join("bin", "llvm-ar"), |
| "-DCMAKE_LINKER=%s" % cipd_dir.join("bin", "ld.lld"), |
| "-DCMAKE_NM=%s" % cipd_dir.join("bin", "llvm-nm"), |
| "-DCMAKE_OBJCOPY=%s" % cipd_dir.join("bin", "llvm-objcopy"), |
| "-DCMAKE_OBJDUMP=%s" % cipd_dir.join("bin", "llvm-objdump"), |
| "-DCMAKE_RANLIB=%s" % cipd_dir.join("bin", "llvm-ranlib"), |
| "-DCMAKE_STRIP=%s" % cipd_dir.join("bin", "llvm-strip"), |
| "-DLLVM_ENABLE_LLD=ON", |
| ] |
| else: |
| extra_options = [ |
| "-DCMAKE_DSYMUTIL=%s" % cipd_dir.join("bin", "dsymutil"), |
| ] |
| |
| # TODO: deduplicate these flags with clang_toolchain recipe. |
| if platform.startswith("linux"): |
| extra_options.extend( |
| [ |
| "-DCMAKE_%s_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread" |
| % mode |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| elif platform.startswith("mac"): |
| extra_options.extend( |
| [ |
| "-DCMAKE_%s_LINKER_FLAGS=-nostdlib++ %s" |
| % (mode, cipd_dir.join("lib", "libc++.a")) |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| elif platform == "fuchsia": |
| extra_options.extend( |
| [ |
| "-DCROSS_TOOLCHAIN_FLAGS_NATIVE=" |
| + "-DCMAKE_SHARED_LINKER_FLAGS=-ldl -lpthread;" |
| + "-DCMAKE_MODULE_LINKER_FLAGS=-ldl -lpthread;" |
| + "-DCMAKE_EXE_LINKER_FLAGS=-ldl -lpthread;" |
| + "-DCMAKE_POLICY_DEFAULT_CMP0056=NEW" |
| ] |
| ) |
| |
| if platform != host_platform: |
| system = platform.split("-")[0].replace("mac", "darwin").capitalize() |
| extra_options.append("-DCMAKE_SYSTEM_NAME=%s" % system) |
| |
| if use_goma: |
| gomacc = api.goma.goma_dir.join("gomacc") |
| extra_options.extend( |
| [ |
| "-DCMAKE_C_COMPILER_LAUNCHER=%s" % gomacc, |
| "-DCMAKE_CXX_COMPILER_LAUNCHER=%s" % gomacc, |
| "-DCMAKE_ASM_COMPILER_LAUNCHER=%s" % gomacc, |
| ] |
| ) |
| |
| with api.context(cwd=build_dir): |
| api.step( |
| "configure %s llvm" % triple, |
| [ |
| cipd_dir.join("bin", "cmake"), |
| "-GNinja", |
| "-DCMAKE_MAKE_PROGRAM=%s" % cipd_dir.join("ninja"), |
| "-DCMAKE_BUILD_TYPE=RelWithDebInfo", |
| "-DCMAKE_INSTALL_PREFIX=", |
| "-DCMAKE_C_COMPILER=%s" % cipd_dir.join("bin", "clang"), |
| "-DCMAKE_C_COMPILER_TARGET=%s" % triple, |
| "-DCMAKE_CXX_COMPILER=%s" % cipd_dir.join("bin", "clang++"), |
| "-DCMAKE_CXX_COMPILER_TARGET=%s" % triple, |
| "-DCMAKE_ASM_COMPILER=%s" % cipd_dir.join("bin", "clang"), |
| "-DCMAKE_ASM_COMPILER_TARGET=%s" % triple, |
| "-DCMAKE_SYSROOT=%s" % sysroot, |
| "-DLLVM_HOST_TRIPLE=%s" % triple, |
| "-DLLVM_TARGETS_TO_BUILD=X86;AArch64", |
| "-DLLVM_DISTRIBUTION_COMPONENTS=" + ";".join(components), |
| "-DLLVM_BUILD_LLVM_DYLIB=ON", |
| "-DLLVM_EXTERNALIZE_DEBUGINFO=ON", |
| "-DLLVM_ENABLE_LIBXML2=OFF", |
| "-DLLVM_ENABLE_TERMINFO=OFF", |
| "-DLLVM_ENABLE_ZLIB=OFF", |
| "-DLLVM_ENABLE_BACKTRACES=OFF", |
| ] |
| + extra_options |
| + [llvm_dir.join("llvm")], |
| ) |
| api.step( |
| "build %s llvm" % triple, |
| [cipd_dir.join("ninja"), "distribution", "-j%s" % ninja_jobs], |
| ) |
| |
| with api.context(env={"DESTDIR": install_dir}): |
| api.step( |
| "install %s llvm" % triple, |
| [cipd_dir.join("ninja"), "install-distribution"], |
| ) |
| |
| if platform == "fuchsia": |
| api.file.copytree( |
| "copy %s libs" % triple, |
| install_dir.join("lib"), |
| pkg_dir.join("arch", target, "lib"), |
| ) |
| |
| if platform == "fuchsia": |
| api.step( |
| "merge headers", |
| cmd=[ |
| "python", |
| api.resource("merge_headers.py"), |
| "--out", |
| pkg_dir.join("pkg", "llvm", "include"), |
| "--def1", |
| "__aarch64__", |
| "--def2", |
| "__x86_64__", |
| staging_dir.join("llvm_x86_64_fuchsia_install_dir", "include"), |
| staging_dir.join("llvm_aarch64_fuchsia_install_dir", "include"), |
| ], |
| ) |
| |
| if platform == "fuchsia": |
| include_dir = pkg_dir.join("pkg", "llvm", "include") |
| else: |
| include_dir = pkg_dir.join("include") |
| step_result = api.file.read_text( |
| "llvm-config.h", |
| include_dir.join("llvm", "Config", "llvm-config.h"), |
| test_data='#define LLVM_VERSION_STRING "7.0.0svn"', |
| ) |
| m = re.search(r'LLVM_VERSION_STRING "([a-zA-Z0-9.-]+)"', step_result) |
| assert m, "Cannot determine LLVM version" |
| llvm_version = m.group(1) |
| |
| api.file.copy("copy license file", llvm_dir.join("llvm", "LICENSE.TXT"), pkg_dir) |
| |
| api.cas_util.upload(pkg_dir) |
| |
| api.cipd_util.upload_package( |
| "fuchsia/third_party/llvm/%s" % platform, |
| pkg_dir, |
| search_tag={"git_revision": revision}, |
| repository=repository, |
| metadata=[("version", llvm_version)], |
| ) |
| |
| |
| def GenTests(api): |
| revision = "75b05681239cb309a23fcb4f8864f177e5aa62da" |
| for platform in PLATFORMS: |
| yield ( |
| api.status_check.test(platform.replace("-", "_")) |
| + api.buildbucket.ci_build(git_repo=LLVM_PROJECT_GIT, revision=revision) |
| + api.properties(platform=platform) |
| + api.step_data( |
| "checkout.git rev-parse", api.raw_io.stream_output_text(revision) |
| ) |
| ) |
| yield ( |
| api.status_check.test("linux_arm64_host") |
| + api.buildbucket.ci_build(git_repo=LLVM_PROJECT_GIT, revision=revision) |
| + api.platform.name("linux") |
| + api.platform.arch("arm") |
| + api.platform.bits(64) |
| + api.properties(platform="linux-arm64") |
| + api.step_data( |
| "checkout.git rev-parse", api.raw_io.stream_output_text(revision) |
| ) |
| ) |
