# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building Clang toolchain."""

from contextlib import contextmanager
from recipe_engine.recipe_api import Property
from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest

from google.protobuf import json_format

import re

DEPS = [
    "fuchsia/git",
    "fuchsia/gitiles",
    "fuchsia/goma",
    "fuchsia/macos_sdk",
    "fuchsia/toolchain",
    "fuchsia/upload",
    "fuchsia/windows_sdk",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/isolated",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/properties",
    "recipe_engine/python",
    "recipe_engine/raw_io",
    "recipe_engine/scheduler",
    "recipe_engine/step",
]

TARGET_TO_ARCH = {
    "x64": "x86_64",
    "arm64": "aarch64",
}
TARGETS = TARGET_TO_ARCH.keys()

PLATFORM_TO_TRIPLE = {
    "linux-amd64": "x86_64-linux-gnu",
    "linux-arm64": "aarch64-linux-gnu",
    "mac-amd64": "x86_64-apple-darwin",
    "windows-amd64": "x86_64-pc-windows-msvc",
}
PLATFORMS = PLATFORM_TO_TRIPLE.keys()

# TODO(fxbug.dev/27210): remove this and switch to normalized triples.
TRIPLE_TO_NORMALIZED_TRIPLE = {
    "x86_64-linux-gnu": "x86_64-unknown-linux-gnu",
    "aarch64-linux-gnu": "aarch64-unknown-linux-gnu",
    "x86_64-apple-darwin": "x86_64-apple-darwin",
    "x86_64-pc-windows-msvc": "x86_64-pc-windows-msvc",
}

LIBXML2_GIT = "https://fuchsia.googlesource.com/third_party/libxml2"
ZLIB_GIT = "https://fuchsia.googlesource.com/third_party/zlib"

CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com"

PROPERTIES = {
    "repository": Property(
        kind=str,
        help="Git repository URL",
        default="https://fuchsia.googlesource.com/third_party/llvm-project",
    ),
    "revision": Property(kind=str, help="Git revision", default=None),
    "platform": Property(kind=str, help="CIPD platform for the target", default=None),
}


def slashes(api, path):
    return path.replace("\\", "/") if api.platform.is_win else path


def build_zlib(api, options, arguments, destdir, ninja_jobs, cipd_dir, manifest):
    zlib_dir = api.path["start_dir"].join("zlib")
    src_dir = zlib_dir.join("src")
    revision = api.git.checkout(ZLIB_GIT, src_dir, ref="upstream/master")
    git_checkout = manifest.directories[str(src_dir)].git_checkout
    git_checkout.repo_url = ZLIB_GIT
    git_checkout.revision = revision
    build_dir = zlib_dir.join("build")
    api.file.ensure_directory("make build dir", build_dir)
    with api.context(cwd=build_dir):
        api.step(
            "configure",
            [cipd_dir.join("bin", "cmake")]
            + [slashes(api, option.format(**arguments)) for option in options]
            + ["-DAMD64=ON", src_dir],
        )
        api.step(
            "build", [cipd_dir.join("ninja"), "-j%d" % ninja_jobs,],
        )
        with api.context(env={"DESTDIR": destdir}):
            api.step(
                "install", [cipd_dir.join("ninja"), "install",],
            )


def build_libxml2(api, options, arguments, destdir, ninja_jobs, cipd_dir, manifest):
    libxml2_dir = api.path["start_dir"].join("libxml2")
    src_dir = libxml2_dir.join("src")
    revision = api.git.checkout(LIBXML2_GIT, src_dir, ref="upstream/master")
    git_checkout = manifest.directories[str(src_dir)].git_checkout
    git_checkout.repo_url = LIBXML2_GIT
    git_checkout.revision = revision
    build_dir = libxml2_dir.join("build")
    api.file.ensure_directory("make build dir", build_dir)
    with api.context(cwd=build_dir):
        api.step(
            "configure",
            [cipd_dir.join("bin", "cmake")]
            + [slashes(api, option.format(**arguments)) for option in options]
            + [
                "-DBUILD_SHARED_LIBS=OFF",
                "-DLIBXML2_WITH_ICU=OFF",
                "-DLIBXML2_WITH_LZMA=OFF",
                "-DLIBXML2_WITH_PYTHON=OFF",
                # TODO(phosek): lld only links libxml2 and not zlib, so when
                # zlib support in libxml2 is enabled, this fails in the case of
                # static linking.
                "-DLIBXML2_WITH_ZLIB=OFF",
            ]
            + [src_dir],
        )
        api.step(
            "build", [cipd_dir.join("ninja"), "-j%d" % ninja_jobs,],
        )
        with api.context(env={"DESTDIR": destdir}):
            api.step(
                "install", [cipd_dir.join("ninja"), "install",],
            )


# //zircon/public/gn/toolchain/clang.gni:clang_runtime sets the JSON schema.
#
# This function makes the prototype spec; debug and breakpad info is added by
# runtimes.py.
def make_runtimes_spec(clang_version):
    # TODO(fxbug.dev/27110): Ideally this would be done by the cmake build itself.
    runtimes = []
    for arch in TARGET_TO_ARCH.itervalues():
        target_triple = "{arch}-unknown-fuchsia".format(arch=arch)
        target = [target_triple, "{arch}-fuchsia".format(arch=arch)]
        runtime_dir = "clang/{version}/lib/{target}".format(
            version=clang_version, target=target_triple
        )
        cxx_lib_dir = "{target}/c++".format(target=target_triple)
        for mode_ldflags in [[], ["-static-libstdc++"]]:
            for mode_cflags, mode_multilib, mode_runtimes in [
                ([], "", []),
                (["-fsanitize=address"], "asan/", ["libclang_rt.asan.so"]),
                (["-fsanitize=undefined"], "", ["libclang_rt.ubsan_standalone.so"]),
            ]:
                runtime = [
                    {"dist": runtime_dir + "/" + soname} for soname in mode_runtimes
                ]
                if not mode_ldflags:
                    cxx_lib = [
                        {
                            "dist": cxx_lib_dir + "/" + mode_multilib + soname,
                            "name": soname.split(".")[0],
                        }
                        for soname in [
                            "libc++.so.2",
                            "libc++abi.so.1",
                            "libunwind.so.1",
                        ]
                    ]
                    runtime.extend(cxx_lib)

                runtimes.append(
                    {
                        "target": target,
                        "cflags": mode_cflags,
                        "ldflags": mode_ldflags,
                        "runtime": runtime,
                    }
                )

    return runtimes


def RunSteps(api, repository, revision, platform):
    use_goma = (
        not api.platform.arch == "arm" and api.platform.bits == 64
    ) and not api.platform.is_win
    if use_goma:
        api.goma.ensure()
        ninja_jobs = api.goma.jobs
        goma_context = lambda: api.goma.build_with_goma()
    else:
        ninja_jobs = api.platform.cpu_count

        @contextmanager
        def null_context():
            yield

        goma_context = null_context

    prod = api.buildbucket.builder_id.bucket == "prod"
    ci = api.buildbucket.builder_id.bucket == "ci"

    gitiles_commit = api.buildbucket.build_input.gitiles_commit
    if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id:
        repository = "https://%s/%s" % (gitiles_commit.host, gitiles_commit.project)
        revision = gitiles_commit.id

    # TODO: factor this out into a host_build recipe module.
    host_platform = "%s-%s" % (
        api.platform.name.replace("win", "windows"),
        {"intel": {32: "386", 64: "amd64",}, "arm": {32: "armv6", 64: "arm64",},}[
            api.platform.arch
        ][api.platform.bits],
    )
    target_platform = platform or host_platform

    use_breakpad = host_platform == "linux-amd64"

    manifest = Manifest()

    with api.step.nest("ensure_packages"):
        with api.context(infra_steps=True):
            cipd_dir = api.path["start_dir"].join("cipd")
            pkgs = api.cipd.EnsureFile()
            # We don't have SDK for linux-arm64 or win, but we only need sysroot.
            if (
                api.platform.arch == "arm" and api.platform.bits == 64
            ) or api.platform.is_win:
                pkgs.add_package("fuchsia/sdk/core/linux-amd64", "latest", "sdk")
            else:
                pkgs.add_package("fuchsia/sdk/core/${platform}", "latest", "sdk")
            if api.platform.name == "win":
                pkgs.add_package("infra/cmake/${platform}", "version:3.16.1")
                pkgs.add_package("infra/ninja/${platform}", "version:1.9.0")
                # TODO(phosek): Switch to CIPD package once we have it.
                api.isolated.download(
                    "msys",
                    isolated_hash="7b23533856ac8a1a2fec614e71df22cc065c7448",
                    output_dir=cipd_dir.join("msys"),
                )
                api.isolated.download(
                    "clang",
                    isolated_hash="fd47a7ec0e608bfc7d08714fdb3ef48585cec60d",
                    output_dir=cipd_dir,
                )
            else:
                pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration")
                if api.platform.arch == "arm" and api.platform.bits == 64:
                    pkgs.add_package(
                        "fuchsia/third_party/cmake/${platform}",
                        "git_revision:4f90fef85f6391e63dfae26bd6f290c59ea48e95",
                    )
                else:
                    # TODO(phosek): Switch to fuchsia/third_party/cmake when 3.19 is out.
                    pkgs.add_package(
                        "infra/cmake/${platform}", "version:3.14.6",
                    )
                pkgs.add_package(
                    "fuchsia/third_party/ninja/${platform}",
                    "git_revision:0ccc7886fd4694ae1372d29b4954e2dd3be118be",
                )
            pkgs.add_package("fuchsia/sysroot/linux-386", "latest", "linux-i386")
            pkgs.add_package("fuchsia/sysroot/linux-amd64", "latest", "linux-amd64")
            pkgs.add_package("fuchsia/sysroot/linux-arm64", "latest", "linux-arm64")
            pkgs.add_package("fuchsia/sysroot/linux-armv6l", "latest", "linux-armhf")
            if use_breakpad:
                pkgs.add_package(
                    "fuchsia/tools/breakpad/${platform}", "latest", "breakpad"
                )
            ensured = api.cipd.ensure(cipd_dir, pkgs)
            for subdir, pins in ensured.iteritems():
                directory = manifest.directories[str(cipd_dir.join(subdir))]
                directory.cipd_server_host = CIPD_SERVER_HOST
                for pin in pins:
                    directory.cipd_package[pin.package].instance_id = pin.instance_id

    staging_dir = api.path["start_dir"].join("staging")
    pkg_dir = staging_dir.join("llvm_install")
    api.file.ensure_directory("create pkg dir", pkg_dir)

    with api.context(infra_steps=True):
        llvm_dir = api.path["start_dir"].join("llvm-project")
        revision = api.git.checkout(
            repository,
            path=llvm_dir,
            ref=revision,
            step_test_data=lambda: api.raw_io.test_api.stream_output(revision),
        )
        git_checkout = manifest.directories[str(llvm_dir)].git_checkout
        git_checkout.repo_url = repository
        git_checkout.revision = revision

    target_triple = PLATFORM_TO_TRIPLE[target_platform]
    host_triple = PLATFORM_TO_TRIPLE[host_platform]

    with api.macos_sdk(kind="ios"), api.windows_sdk(), goma_context():
        if api.platform.name == "linux":
            host_sysroot = cipd_dir.join(host_platform)
            target_sysroot = cipd_dir.join(target_platform)
        elif api.platform.name == "mac":
            # TODO(fxbug.dev/3043): Eventually use our own hermetic sysroot as for Linux.
            step_result = api.step(
                "xcrun",
                ["xcrun", "--sdk", "macosx", "--show-sdk-path"],
                stdout=api.raw_io.output(name="sdk-path", add_output_log=True),
                step_test_data=lambda: api.raw_io.test_api.stream_output(
                    "/some/xcode/path"
                ),
            )
            target_sysroot = host_sysroot = step_result.stdout.strip()
        elif api.platform.name == "win":
            target_sysroot = host_sysroot = api.windows_sdk.sdk_dir
        else:  # pragma: no cover
            assert False, "unsupported platform"

        arguments = {
            "target_triple": TRIPLE_TO_NORMALIZED_TRIPLE[target_triple],
            "host_triple": host_triple,
            "target_sysroot": target_sysroot,
            "host_sysroot": host_sysroot,
            "linux_arm64_sysroot": cipd_dir.join("linux-arm64"),
            "linux_armhf_sysroot": cipd_dir.join("linux-armhf"),
            "linux_i386_sysroot": cipd_dir.join("linux-i386"),
            "linux_amd64_sysroot": cipd_dir.join("linux-amd64"),
            "fuchsia_sdk": cipd_dir.join("sdk"),
        }

        if api.platform.is_win:
            arguments.update(
                {
                    "cc": cipd_dir.join("bin", "clang-cl.exe"),
                    "cxx": cipd_dir.join("bin", "clang-cl.exe"),
                    "ar": cipd_dir.join("bin", "llvm-ar.exe"),
                    "ld": cipd_dir.join("bin", "lld-link.exe"),
                    "mt": cipd_dir.join("bin", "llvm-mt.exe"),
                    "nm": cipd_dir.join("bin", "llvm-nm.exe"),
                    "objcopy": cipd_dir.join("bin", "llvm-objcopy.exe"),
                    "objdump": cipd_dir.join("bin", "llvm-objdump.exe"),
                    "ranlib": cipd_dir.join("bin", "llvm-ranlib.exe"),
                    "rc": cipd_dir.join("bin", "llvm-rc.exe"),
                    "readelf": cipd_dir.join("bin", "llvm-readelf.exe"),
                    "strip": cipd_dir.join("bin", "llvm-strip.exe"),
                    "ninja": cipd_dir.join("ninja.exe"),
                }
            )
        else:
            arguments.update(
                {
                    "cc": cipd_dir.join("bin", "clang"),
                    "cxx": cipd_dir.join("bin", "clang++"),
                    "ar": cipd_dir.join("bin", "llvm-ar"),
                    "ld": cipd_dir.join("bin", "ld.lld"),
                    "lipo": cipd_dir.join("bin", "llvm-lipo"),
                    "nm": cipd_dir.join("bin", "llvm-nm"),
                    "objcopy": cipd_dir.join("bin", "llvm-objcopy"),
                    "objdump": cipd_dir.join("bin", "llvm-objdump"),
                    "ranlib": cipd_dir.join("bin", "llvm-ranlib"),
                    "readelf": cipd_dir.join("bin", "llvm-readelf"),
                    "strip": cipd_dir.join("bin", "llvm-strip"),
                    "ninja": cipd_dir.join("ninja"),
                }
            )
        if use_goma:
            arguments.update(
                {"gomacc": api.goma.goma_dir.join("gomacc"),}
            )

        options = [
            "-GNinja",
            "-DCMAKE_MAKE_PROGRAM={ninja}",
            "-DCMAKE_INSTALL_PREFIX=",
            "-DCMAKE_C_COMPILER={cc}",
            "-DCMAKE_CXX_COMPILER={cxx}",
            "-DCMAKE_ASM_COMPILER={cc}",
        ]
        if use_goma:
            options.extend(
                [
                    "-DCMAKE_C_COMPILER_LAUNCHER={gomacc}",
                    "-DCMAKE_CXX_COMPILER_LAUNCHER={gomacc}",
                    "-DCMAKE_ASM_COMPILER_LAUNCHER={gomacc}",
                ]
            )

        options.extend(
            {
                "linux": [
                    "-DCMAKE_AR={ar}",
                    "-DCMAKE_LINKER={ld}",
                    "-DCMAKE_NM={nm}",
                    "-DCMAKE_OBJCOPY={objcopy}",
                    "-DCMAKE_OBJDUMP={objdump}",
                    "-DCMAKE_RANLIB={ranlib}",
                    "-DCMAKE_READELF={readelf}",
                    "-DCMAKE_STRIP={strip}",
                ],
                "mac": ["-DCMAKE_LIPO={lipo}",],
                "win": [
                    "-DCMAKE_AR={ar}",
                    "-DCMAKE_LINKER={ld}",
                    "-DCMAKE_NM={nm}",
                    "-DCMAKE_OBJCOPY={objcopy}",
                    "-DCMAKE_OBJDUMP={objdump}",
                    "-DCMAKE_RANLIB={ranlib}",
                    "-DCMAKE_READELF={readelf}",
                    "-DCMAKE_STRIP={strip}",
                    # TODO(phosek): reenable once we update the host toolchain
                    # "-DCMAKE_RC_COMPILER={rc}",
                    # "-DCMAKE_MT={mt}",
                ],
            }[api.platform.name]
        )

        options.extend(["-DCMAKE_SYSROOT={target_sysroot}"])

        if not api.platform.is_win:
            # TODO(phosek): Build these even for Windows.
            with api.step.nest("zlib"):
                zlib_install_dir = staging_dir.join("zlib_install")
                api.file.ensure_directory("create zlib_install_dir", zlib_install_dir)
                build_zlib(
                    api,
                    options
                    + [
                        "-DCMAKE_%s_FLAGS=-O3 -fPIC --target=%s" % (lang, target_triple)
                        for lang in ["C", "CXX"]
                    ],
                    arguments,
                    zlib_install_dir,
                    ninja_jobs,
                    cipd_dir,
                    manifest,
                )
            with api.step.nest("libxml2"):
                libxml2_install_dir = staging_dir.join("libxml2_install")
                api.file.ensure_directory(
                    "create libxml2_install_dir", libxml2_install_dir
                )
                build_libxml2(
                    api,
                    options
                    + [
                        "-DCMAKE_%s_FLAGS=-O3 -fPIC --target=%s" % (lang, target_triple)
                        for lang in ["C", "CXX"]
                    ],
                    arguments,
                    libxml2_install_dir,
                    ninja_jobs,
                    cipd_dir,
                    manifest,
                )

        json_manifest = json_format.MessageToJson(
            manifest, preserving_proto_field_name=True
        )
        api.file.write_text(
            "source manifest", pkg_dir.join("source_manifest.json"), json_manifest
        )

        # build clang+llvm
        build_dir = staging_dir.join("llvm_build")
        api.file.ensure_directory("create llvm build dir", build_dir)

        arguments.update(
            {
                "BOOTSTRAP_": "BOOTSTRAP_",
                "STAGE2_": "STAGE2_",
                "stage2_": "stage2-",
                "_stage2": "",
            }
            if prod
            else {"BOOTSTRAP_": "", "STAGE2_": "", "stage2_": "", "_stage2": "-stage2",}
        )

        llvm_projects = ["clang", "clang-tools-extra", "lld", "llvm"]
        llvm_runtimes = ["compiler-rt", "libcxx", "libcxxabi", "libunwind"]

        if not api.platform.is_win:
            options.extend(
                [
                    "-D{BOOTSTRAP_}ZLIB_INCLUDE_DIR=%s"
                    % zlib_install_dir.join("include"),
                    "-D{BOOTSTRAP_}ZLIB_LIBRARY=%s"
                    % zlib_install_dir.join("lib", "libz.a"),
                    "-D{BOOTSTRAP_}LIBXML2_INCLUDE_DIR=%s"
                    % libxml2_install_dir.join("include", "libxml2"),
                    "-D{BOOTSTRAP_}LIBXML2_LIBRARY=%s"
                    % libxml2_install_dir.join("lib", "libxml2.a"),
                ]
            )
            if prod:
                options.extend(
                    [
                        "-DZLIB_INCLUDE_DIR=%s" % zlib_install_dir.join("include"),
                        "-DZLIB_LIBRARY=%s" % zlib_install_dir.join("lib", "libz.a"),
                        "-DLIBXML2_INCLUDE_DIR=%s"
                        % libxml2_install_dir.join("include", "libxml2"),
                        "-DLIBXML2_LIBRARY=%s"
                        % libxml2_install_dir.join("lib", "libxml2.a"),
                    ]
                )

        if api.platform.name == "linux":
            if prod:
                options.extend(
                    ["-D{BOOTSTRAP_}CMAKE_SYSROOT={target_sysroot}"]
                    + [
                        # BOOTSTRAP_ prefixed flags are passed to the second stage compiler.
                        "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-static-libstdc++" % mode
                        for mode in ["SHARED", "MODULE", "EXE"]
                    ]
                    + [
                        # Unprefixed flags are used by the first stage compiler.
                        "-DCMAKE_%s_LINKER_FLAGS=-static-libstdc++" % mode
                        for mode in ["SHARED", "MODULE", "EXE"]
                    ]
                )
            else:
                options.extend(
                    [
                        # BOOTSTRAP_ prefixed flags are passed to the second stage compiler.
                        "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-static-libstdc++" % mode
                        for mode in ["SHARED", "MODULE", "EXE"]
                    ]
                    + ["-DLLVM_ENABLE_LTO=OFF",]
                )
            if host_triple != target_triple:  # pragma: no cover
                options.extend(
                    [
                        "-D{BOOTSTRAP_}CMAKE_SYSTEM_NAME=Linux",
                        "-D{BOOTSTRAP_}CMAKE_C_COMPILER_TARGET={target_triple}",
                        "-D{BOOTSTRAP_}CMAKE_CXX_COMPILER_TARGET={target_triple}",
                        "-D{BOOTSTRAP_}LLVM_DEFAULT_TARGET_TRIPLE={target_triple}",
                    ]
                )
        elif api.platform.name == "mac":
            if prod:
                options.extend(
                    ["-D{BOOTSTRAP_}CMAKE_SYSROOT={target_sysroot}"]
                    + [
                        # BOOTSTRAP_ prefixed flags are passed to the second stage compiler.
                        "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-nostdlib++ %s"
                        % (mode, build_dir.join("lib", "libc++.a"))
                        for mode in ["SHARED", "MODULE", "EXE"]
                    ]
                    + [
                        # Unprefixed flags are used by the first stage compiler.
                        "-DCMAKE_%s_LINKER_FLAGS=-nostdlib++ %s"
                        % (mode, cipd_dir.join("lib", "libc++.a"))
                        for mode in ["SHARED", "MODULE", "EXE"]
                    ]
                )
            else:
                options.extend(
                    [
                        # BOOTSTRAP_ prefixed flags are passed to the second stage compiler.
                        "-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-nostdlib++ %s"
                        % (mode, cipd_dir.join("lib", "libc++.a"))
                        for mode in ["SHARED", "MODULE", "EXE"]
                    ]
                    + ["-DLLVM_ENABLE_LTO=OFF"]
                )
        elif api.platform.name == "win":
            if not prod:
                options.extend(["-DLLVM_ENABLE_LTO=OFF"])

        # STAGE2_ prefixed flags are passed to the second stage by the first stage build.
        if ci:
            options.extend(
                ["-D{STAGE2_}LLVM_ENABLE_ASSERTIONS=ON",]
            )

        options.extend(
            [
                "-D{STAGE2_}LINUX_aarch64-unknown-linux-gnu_SYSROOT={linux_arm64_sysroot}",
                "-D{STAGE2_}LINUX_armv7-unknown-linux-gnueabihf_SYSROOT={linux_armhf_sysroot}",
                "-D{STAGE2_}LINUX_i386-unknown-linux-gnu_SYSROOT={linux_i386_sysroot}",
                "-D{STAGE2_}LINUX_x86_64-unknown-linux-gnu_SYSROOT={linux_amd64_sysroot}",
                "-D{STAGE2_}FUCHSIA_SDK={fuchsia_sdk}",
            ]
        )

        if api.platform.is_win:
            env_prefixes = {"PATH": [cipd_dir.join("msys")]}
        else:
            env_prefixes = {}

        with api.step.nest("clang"), api.context(
            cwd=build_dir, env_prefixes=env_prefixes
        ):
            api.step(
                "configure",
                [cipd_dir.join("bin", "cmake")]
                + [slashes(api, option.format(**arguments)) for option in options]
                + [
                    "-C",
                    llvm_dir.join(
                        "clang",
                        "cmake",
                        "caches",
                        "Fuchsia{_stage2}.cmake".format(**arguments),
                    ),
                    llvm_dir.join("llvm"),
                ],
            )
            # Build the full (two-stage) distribution.
            api.step(
                "build",
                [
                    cipd_dir.join("ninja"),
                    # This only applies to the first stage, second stage is invoked by
                    # CMake as a subprocess and will use Ninja's default.
                    "-j%d" % ninja_jobs,
                    "{stage2_}distribution".format(**arguments),
                ],
            )
            # Run the tests.
            projects = ["llvm", "clang", "lld"]
            # TODO(phosek): run runtime tests
            #   + [(runtime + '-{target_triple}').format(**arguments)
            #      for runtime in ['unwind', 'cxxabi', 'cxx']]
            api.step(
                "test",
                [cipd_dir.join("ninja"), "-j%d" % api.goma.jobs]
                + [
                    ("{stage2_}check-" + project).format(**arguments)
                    for project in projects
                ],
            )
            with api.context(env={"DESTDIR": pkg_dir}):
                api.step(
                    "install",
                    [
                        cipd_dir.join("ninja"),
                        "{stage2_}install-distribution-stripped".format(**arguments),
                    ],
                )

    step_result = api.file.read_text(
        "Version.inc",
        build_dir.join(
            *(
                (["tools", "clang", "stage2-bins"] if prod else [])
                + ["tools", "clang", "include", "clang", "Basic", "Version.inc"]
            )
        ),
        test_data='#define CLANG_VERSION_STRING "8.0.0"',
    )
    m = re.search(r'CLANG_VERSION_STRING "([a-zA-Z0-9.-]+)"', step_result)
    assert m, "Cannot determine Clang version"
    clang_version = m.group(1)

    api.toolchain.strip_runtimes(
        "generate runtime.json",
        spec=make_runtimes_spec(clang_version),
        path=pkg_dir.join("lib"),
        build_id_subpath="debug/.build-id",
        readelf=cipd_dir.join("bin", "llvm-readelf"),
        dump_syms=(
            cipd_dir.join("breakpad", "dump_syms", "dump_syms")
            if use_breakpad
            else None
        ),
    )

    api.python(
        "generate license",
        api.resource("generate_license.py"),
        args=["--include"]
        + [
            llvm_dir.join(project, "LICENSE.TXT")
            for project in llvm_projects + llvm_runtimes
        ]
        + (
            [
                "--extract",
                api.path["start_dir"].join("libxml2", "src", "Copyright"),
                "-",
                "--extract",
                api.path["start_dir"].join("zlib", "src", "zlib.h"),
                "4-22",
            ]
            if not api.platform.is_win
            else []
        ),
        stdout=api.raw_io.output(leak_to=pkg_dir.join("LICENSE")),
    )

    isolated = api.upload.upload_isolated(pkg_dir)

    if prod:
        # The published package has the same name for every platform.
        api.upload.cipd_package(
            "fuchsia/third_party/clang/%s" % target_platform,
            pkg_dir,
            [api.upload.DirectoryPath(pkg_dir)],
            {"git_revision": revision},
            repository=repository,
            extra_tags={"version": clang_version},
        )

        # TODO(phosek): move this logic to clang_trigger.py recipe.
        if not target_platform in ["linux-arm64", "windows-amd64"]:
            host = "fuchsia.googlesource.com"
            project = "fuchsia"
            refs = api.gitiles.refs("https://%s/%s" % (host, project))
            ref = "refs/heads/master"
            fuchsia_rev = refs.get(ref, "HEAD")

            # Do a full integration build. This will use the just-built toolchain
            # to build all of Fuchsia to check whether there are any regressions.
            api.scheduler.emit_trigger(
                api.scheduler.BuildbucketTrigger(
                    properties={
                        "clang_toolchain": {
                            # Purely for informational purposes; not consumed by
                            # any recipe or recipe module.
                            "git_repository": repository,
                            "git_revision": revision,
                        },
                        "build.clang_toolchain": {
                            "type": "isolated",
                            "instance": isolated,
                        },
                        "checkout.buildset": "commit/gitiles/%s/%s/+/%s"
                        % (host, project, fuchsia_rev),
                    },
                    tags={
                        "buildset": "commit/gitiles/%s/+/%s"
                        % (repository.split("://")[1], revision),
                        "gitiles_ref": ref,
                    },
                ),
                project="fuchsia",
                jobs={
                    "linux-amd64": (
                        "clang_toolchain.fuchsia-arm64-debug",
                        "clang_toolchain.fuchsia-arm64-release",
                        "clang_toolchain.fuchsia-x64-debug",
                        "clang_toolchain.fuchsia-x64-release",
                    ),
                    "mac-amd64": ("clang_toolchain.fuchsia-host-mac",),
                }[target_platform],
            )


def GenTests(api):
    for os in ("linux", "mac"):
        yield (
            api.test("ci-%s-x64" % os)
            + api.buildbucket.ci_build(
                project="fuchsia",
                bucket="ci",
                git_repo="https://fuchsia.googlesource.com/third_party/llvm-project",
                revision="a" * 40,
            )
            + api.platform.name(os)
            + api.properties(platform=os + "-amd64")
        )

        yield (
            api.test("prod-%s-x64" % os)
            + api.buildbucket.ci_build(
                project="fuchsia",
                bucket="prod",
                git_repo="https://fuchsia.googlesource.com/third_party/llvm-project",
                revision="a" * 40,
            )
            + api.platform.name(os)
            + api.properties(platform=os + "-amd64")
            + api.gitiles.refs("refs", ("refs/heads/master", "b" * 40))
        )

    yield (
        api.test("windows-amd64")
        + api.buildbucket.ci_build(
            project="fuchsia",
            bucket="ci",
            git_repo="https://fuchsia.googlesource.com/third_party/llvm-project",
            revision="a" * 40,
        )
        + api.platform.name("win")
        + api.properties(platform="windows-amd64")
    )

    yield (
        api.test("linux-arm64")
        + api.buildbucket.ci_build(
            project="fuchsia",
            bucket="ci",
            git_repo="https://fuchsia.googlesource.com/third_party/llvm-project",
            revision="a" * 40,
        )
        + api.platform.name("linux")
        + api.platform.arch("arm")
        + api.platform.bits(64)
        + api.properties(platform="linux-arm64")
    )