recipes/contrib/ninja.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
"""Recipe for building Ninja."""

from __future__ import annotations

from collections.abc import Iterator
from dataclasses import dataclass

from recipe_engine import config_types, post_process, recipe_api, recipe_test_api

from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest
from PB.recipes.fuchsia.contrib.ninja import InputProperties

from RECIPE_MODULES.fuchsia import (
    buildbucket_util as buildbucket_util_module,
    cas_util as cas_util_module,
    cipd_util as cipd_util_module,
    cmake as cmake_module,
    git as git_module,
    git_checkout as git_checkout_module,
    macos_sdk as macos_sdk_module,
    make as make_module,
    ninja as ninja_module,
    platform_util as platform_util_module,
    raw_result as raw_result_module,
    uploader_util as uploader_util_module,
    windows_sdk as windows_sdk_module,
)
from RECIPE_MODULES.fuchsia.platform_util.api import Platform
from RECIPE_MODULES.fuchsia.raw_result.api import wrap_run_steps
from RECIPE_MODULES.recipe_engine import (
    buildbucket as buildbucket_module,
    cipd as cipd_module,
    context as context_module,
    file as file_module,
    path as path_module,
    platform as platform_module,
    properties as properties_module,
    proto as proto_module,
    step as step_module,
)


@dataclass
class DEPS(recipe_api.RecipeScriptApi):
    buildbucket: buildbucket_module.API
    cas_util: cas_util_module.API
    cipd: cipd_module.API
    cipd_util: cipd_util_module.API
    cmake: cmake_module.API
    context: context_module.API
    file: file_module.API
    git_checkout: git_checkout_module.API
    macos_sdk: macos_sdk_module.API
    make: make_module.API
    ninja: ninja_module.API
    path: path_module.API
    platform: platform_module.API
    platform_util: platform_util_module.API
    proto: proto_module.API
    raw_result: raw_result_module.API
    step: step_module.API
    uploader_util: uploader_util_module.API
    windows_sdk: windows_sdk_module.API


@dataclass
class TEST_DEPS(recipe_test_api.RecipeTestApi):
    buildbucket_util: buildbucket_util_module.TEST_API
    cipd: cipd_module.TEST_API
    git: git_module.TEST_API
    platform: platform_module.TEST_API
    properties: properties_module.TEST_API


PROPERTIES = InputProperties

GIT_URL = "https://fuchsia.googlesource.com/third_party/github.com/ninja-build/ninja"
GIT_BRANCH = "main"

CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com"

GOOGLETEST_GIT_URL = (
    "https://fuchsia.googlesource.com/third_party/github.com/google/googletest"
)
GOOGLETEST_REVISION = "refs/tags/v1.14.0"

# On select platforms, link the Ninja executable against rpmalloc for a small 10% speed boost.
RPMALLOC_GIT_URL = (
    "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc"
)
# Note third party dependencies should "bake" for ~2 weeks to reduce security
# risk of changes that have not been fully evaluated.
RPMALLOC_REVISION = "refs/tags/1.4.4"

ZLIB_GIT_URL = "https://fuchsia.googlesource.com/third_party/github.com/madler/zlib"
# Note third party dependencies should "bake" for ~2 weeks to reduce security
# risk of changes that have not been fully evaluated.
ZLIB_REVISION = "refs/tags/v1.3.1"

# Used to convert os and arch strings to rpmalloc format
RPMALLOC_MAP = {
    "intel": "x86-64",
    "amd64": "x86-64",
    "arm": "arm64",
    "mac": "macos",
}


@wrap_run_steps
def RunSteps(api: DEPS, props: InputProperties) -> None:
    host_platform = api.platform_util.host
    target_platform = (
        api.platform_util.platform(props.platform) if props.platform else host_platform
    )

    manifest = Manifest()
    src_dir, revision = api.git_checkout(GIT_URL, fallback_ref=GIT_BRANCH)
    package_name = f"fuchsia/third_party/ninja/{target_platform}"
    upload_pkg = api.buildbucket.build.builder.bucket == "prod"
    result = api.uploader_util.prepare_checkout(
        pkg_name=package_name,
        checkout_dir=src_dir,
        current_revision=revision,
        ref=GIT_BRANCH,
        min_baking_delay_days=props.min_baking_delay_days,
        force_build=props.force_build,
        upload_package=upload_pkg,
    )
    if not result.should_build:
        return
    revision = result.commit
    git_checkout = manifest.directories[str(src_dir)].git_checkout
    git_checkout.repo_url = GIT_URL
    git_checkout.revision = revision
    version_data = api.file.read_text(
        "read ninja revision",
        src_dir.joinpath("src", "version.cc"),
        test_data='\nconst char* kNinjaVersion = "1.10.2.git";\n',
    )
    ninja_version = read_revision(api, version_data)
    api.step.empty("ninja version", step_text=ninja_version)

    with api.step.nest("ensure packages"), api.context(infra_steps=True):
        cipd_dir = api.path.start_dir / "cipd"
        pkgs = api.cipd.EnsureFile()
        pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration")
        if api.platform.is_linux:
            pkgs.add_package(
                "fuchsia/third_party/sysroot/linux",
                "integration",
                "sysroot",
            )
        ensured = api.cipd.ensure(cipd_dir, pkgs)
        for subdir, pins in ensured.items():
            directory = manifest.directories[str(cipd_dir / subdir)]
            directory.cipd_server_host = CIPD_SERVER_HOST
            for pin in pins:
                directory.cipd_package[pin.package].instance_id = pin.instance_id

    api.file.write_proto(
        "source manifest",
        src_dir / "source_manifest.json",
        manifest,
        api.proto.JSONPB,
    )

    with api.step.nest("googletest"):
        googletest_src_dir, _ = api.git_checkout(
            GOOGLETEST_GIT_URL, fallback_ref=GOOGLETEST_REVISION
        )
        googletest_install_dir = api.path.start_dir / "googletest_install"

        arguments = {}
        with api.macos_sdk(), api.windows_sdk():
            arguments = get_build_config(api, cipd_dir, target_platform)
            options = get_cmake_options(target_platform, host_platform)

            # Avoid a warning when compiling GoogleTest.
            options.append("-DCMAKE_CXX_FLAGS=-Wno-character-conversion")

            # Ensure everything is installed directly under the installation
            # directory, and not DESTDIR/usr/local on Unix and
            # c;/Program Files/${PROJECT_NAME} on Windows.
            options.append(f"-DCMAKE_INSTALL_PREFIX={googletest_install_dir}")

            api.cmake.build_with_ninja(
                src_dir=googletest_src_dir,
                build_type="Release",
                extra_args=[option.format(**arguments) for option in options],
            )

    use_zlib = api.platform.is_linux or api.platform.is_mac
    if use_zlib:
        with api.step.nest("zlib"):
            zlib_src_dir, _ = api.git_checkout(ZLIB_GIT_URL, fallback_ref=ZLIB_REVISION)
            with api.macos_sdk(), api.windows_sdk():
                config = get_build_config(api, cipd_dir, target_platform)
                triple = f"--target={target_platform.triple}"
                env = {
                    "CC": config["cc"],
                    "CXX": config["cxx"],
                    "AR": f"{cipd_dir.joinpath('bin', 'llvm-ar')}",
                    "CFLAGS": f"{triple} --sysroot={config['sysroot']}",
                }
                env["LDFLAGS"] = triple
                if "link_flags" in config:
                    env["LDFLAGS"] += f" {config['link_flags']}"

                with (
                    api.step.nest(f"build zlib-static-{api.platform.name}"),
                    api.context(env=env, cwd=zlib_src_dir),
                ):
                    api.step(
                        "configure",
                        [
                            "./configure",
                            "--static",
                            "--64",
                        ],
                    )
                    api.make(zlib_src_dir)
                    zlib_static_lib = zlib_src_dir / "libz.a"

    use_rpmalloc = api.platform.is_linux or api.platform.is_mac
    if use_rpmalloc:
        with api.step.nest("rpmalloc"):
            rpmalloc_src_dir, _ = api.git_checkout(
                RPMALLOC_GIT_URL, fallback_ref=RPMALLOC_REVISION
            )

            # Patch configure.py to replace -Werror with -Wno-error because
            # it is causing too many compilation issues when building rpmalloc
            # with this recipe.
            build_ninja_clang_path = rpmalloc_src_dir / "build" / "ninja" / "clang.py"
            build_ninja_clang_py = api.file.read_text(
                f"read {build_ninja_clang_path}",
                build_ninja_clang_path,
                "CXXFLAGS = ['-Wno-disabled-macro-expansion']",
            )
            build_ninja_clang_py = build_ninja_clang_py.replace(
                "'-Werror',", "'-Wno-error',"
            )
            api.file.write_text(
                f"write {build_ninja_clang_path}",
                build_ninja_clang_path,
                build_ninja_clang_py,
            )

            with api.macos_sdk(), api.windows_sdk():
                rpmalloc_os, rpmalloc_arch = api.platform.name, api.platform.arch
                rpmalloc_os = RPMALLOC_MAP.get(rpmalloc_os, rpmalloc_os)
                rpmalloc_arch = RPMALLOC_MAP.get(rpmalloc_arch, rpmalloc_arch)

                config = get_build_config(api, cipd_dir, target_platform)
                triple = f"--target={target_platform.triple}"
                env = {
                    "CC": config["cc"],
                    "CXX": config["cxx"],
                    "AR": f"{cipd_dir.joinpath('bin', 'llvm-ar')}",
                    "CFLAGS": f"{triple} --sysroot={config['sysroot']}",
                }
                env["LDFLAGS"] = triple
                if "link_flags" in config:
                    env["LDFLAGS"] += f" {config['link_flags']}"

                with (
                    api.step.nest(f"build rpmalloc-{api.platform.name}"),
                    api.context(env=env, cwd=rpmalloc_src_dir),
                ):
                    api.step(
                        "configure",
                        [
                            "python3",
                            rpmalloc_src_dir / "configure.py",
                            "-c",
                            "release",
                            "-a",
                            rpmalloc_arch,
                            "--lto",
                        ],
                    )

                    # NOTE: Only build the static library.
                    if rpmalloc_os == "macos":
                        # For MacOS, the library path does not include the architecture,
                        # Probably because it can contain multi-arch binaries.
                        rpmalloc_static_lib = api.path.join(
                            "lib",
                            rpmalloc_os,
                            "release",
                            "librpmallocwrap.a",
                        )
                    else:
                        rpmalloc_static_lib = api.path.join(
                            "lib",
                            rpmalloc_os,
                            "release",
                            rpmalloc_arch,
                            "librpmallocwrap.a",
                        )
                    api.ninja("ninja", [rpmalloc_static_lib])

    staging_dir = api.path.mkdtemp("ninja")
    build_dir = staging_dir / "ninja_build_dir"
    api.file.ensure_directory("create build dir", build_dir)
    pkg_dir = staging_dir / "ninja"
    api.file.ensure_directory("create pkg dir", pkg_dir)
    cmake_src_dir = src_dir

    if use_rpmalloc:
        # Building Ninja with rpmalloc without modifying the upstream project.
        # See https://github.com/ninja-build/ninja/pull/2071
        # In a nutshell, generate a tiny CMakeList.txt file in a different directory
        cmake_src_dir = staging_dir / "rpmalloc_link_src_dir"
        api.file.ensure_directory("create rpmalloc_link source dir", cmake_src_dir)
        cmakelists_txt_file = cmake_src_dir / "CMakeLists.txt"
        cmake_ninja_src_dir = api.path.relpath(src_dir, cmake_src_dir)
        cmake_rpmalloc_lib = api.path.abspath(rpmalloc_src_dir / rpmalloc_static_lib)
        api.file.write_text(
            "generate rpmalloc_link CMakeLists.txt",
            cmakelists_txt_file,
            f"""cmake_minimum_required(VERSION 3.15)
project(ninja-rpmalloc)
include(CTest)
add_subdirectory("{cmake_ninja_src_dir}" ninja)
target_link_libraries(ninja PRIVATE "{cmake_rpmalloc_lib}" -lpthread -ldl)
""",
        )

    arguments = {}
    with api.macos_sdk(), api.windows_sdk():
        arguments = get_build_config(api, cipd_dir, target_platform)
        options = get_cmake_options(target_platform, host_platform)

        if "link_flags" in arguments:
            options.extend(
                [
                    "-DCMAKE_%s_LINKER_FLAGS={link_flags}" % mode
                    for mode in ["SHARED", "MODULE", "EXE"]
                ]
            )

        # Ensure that `ninja test` invoked below prints usable output to
        # stdout. Using --verbose is preferred to --output-on-failure
        # to verify the list of unit tests that were really run.
        options.append("-DCMAKE_CTEST_ARGUMENTS=--verbose")

        if use_zlib:
            options.extend(
                [
                    f"-DZLIB_INCLUDE_DIR={zlib_src_dir}",
                    f"-DZLIB_LIBRARY={zlib_static_lib}",
                ]
            )

        # Ensure local GoogleTest sources are used. otherwise the CMakeList.txt
        # file will download an archive from the network.
        options.append(f"-DGTEST_ROOT={googletest_install_dir}")

        api.cmake.build_with_ninja(
            src_dir=cmake_src_dir,
            build_dir=build_dir,
            build_type="Release",
            extra_args=[option.format(**arguments) for option in options],
            install_cmd="install" if api.platform.is_win else "install/strip",
            install_dir=pkg_dir,
        )
        if target_platform == host_platform:
            api.ninja("test", ["test"], build_dir=build_dir)

    ninja = pkg_dir.joinpath("bin", f"ninja{'.exe' if api.platform.is_win else ''}")

    # TODO(fxbug.dev/86997): Remove the duplicated ninja binary after transition completes
    copied_ninja = pkg_dir.joinpath("ninja" + (".exe" if api.platform.is_win else ""))
    api.file.copy("duplicate ninja binary", ninja, copied_ninja)

    # Upload the installation to CAS.
    api.cas_util.upload(pkg_dir, [ninja, copied_ninja], output_property="isolated")

    if upload_pkg:
        # Upload the installation to CIPD for production builds.
        api.cipd_util.upload_package(
            package_name,
            pkg_dir,
            pkg_paths=[ninja, copied_ninja],
            search_tag={"git_revision": revision},
            repository=GIT_URL,
            metadata=[("version", ninja_version)],
        )


def read_revision(api: DEPS, source: str) -> str:
    for curline in source.splitlines():
        if "kNinjaVersion" in curline:
            revision = curline[curline.find("=") + 1 : curline.find(";")].strip(' "')
            return revision
    raise api.step.StepFailure("revision string not found")  # pragma no cover


def get_sysroot(
    api: DEPS, cipd_dir: config_types.Path, platform: Platform
) -> config_types.Path:
    if platform.is_linux:
        return cipd_dir / "sysroot"
    if platform.is_mac:
        return api.macos_sdk.sysroot
    raise api.step.StepFailure("unsupported platform")  # pragma: no cover


def get_build_config(
    api: DEPS, cipd_dir: config_types.Path, platform: Platform
) -> dict[str, config_types.Path | str]:
    config: dict[str, config_types.Path | str] = {
        "ninja": api.ninja.path,
    }
    if platform.is_win:
        config.update(
            {
                "cc": cipd_dir.joinpath("bin", "clang-cl.exe"),
                "cxx": cipd_dir.joinpath("bin", "clang-cl.exe"),
                "sysroot": api.windows_sdk.sdk_dir,
            }
        )
    else:
        config.update(
            {
                "cc": cipd_dir.joinpath("bin", "clang"),
                "cxx": cipd_dir.joinpath("bin", "clang++"),
                "sysroot": get_sysroot(api, cipd_dir, platform),
            }
        )
    if platform.is_mac:
        config["link_flags"] = f"-nostdlib++ {cipd_dir.joinpath('lib', 'libc++.a')}"

    return config


def get_cmake_options(target_platform: Platform, host_platform: Platform) -> list[str]:
    options = [
        "-DCMAKE_C_COMPILER={cc}",
        "-DCMAKE_CXX_COMPILER={cxx}",
        "-DCMAKE_ASM_COMPILER={cc}",
        "-DCMAKE_MAKE_PROGRAM={ninja}",
        "-DCMAKE_INSTALL_PREFIX=",
        "-DCMAKE_SYSROOT={sysroot}",
    ]

    if target_platform != host_platform:
        options.extend(
            [
                f"-DCMAKE_C_COMPILER_TARGET={target_platform.triple}",
                f"-DCMAKE_CXX_COMPILER_TARGET={target_platform.triple}",
                f"-DCMAKE_ASM_COMPILER_TARGET={target_platform.triple}",
                f"-DCMAKE_SYSTEM_NAME={target_platform.os.replace('mac', 'darwin').title()}",
            ]
        )

    return options


def GenTests(api: TEST_DEPS) -> Iterator[recipe_test_api.TestData]:
    for plat in ("linux", "mac", "win"):
        yield (
            api.buildbucket_util.test(
                plat, bucket="prod", git_repo=GIT_URL, revision="a" * 40
            )
            + api.platform.name(plat)
        )

    yield (
        api.buildbucket_util.test(
            "mac-arm64", bucket="prod", git_repo=GIT_URL, revision="a" * 40
        )
        + api.platform.name("mac")
        + api.properties(platform="mac-arm64")
    )

    yield (
        api.buildbucket_util.test(
            "baking_delay_up_to_date",
            bucket="prod",
            git_repo=GIT_URL,
            revision="a" * 40,
        )
        + api.platform.name("linux")
        + api.properties(min_baking_delay_days=14)
        + api.git.get_baked_commit(
            "prepare uploader checkout.get baked commit",
            "b" * 40,
        )
        + api.step_data(
            f"prepare uploader checkout.check package up-to-date.cipd search fuchsia/third_party/ninja/linux-amd64 git_revision:{'b' * 40}",
            api.cipd.example_search("fuchsia/third_party/ninja/linux-amd64"),
        )
        + api.post_process(
            post_process.MustRun,
            "prepare uploader checkout.check package up-to-date",
        )
        + api.post_process(post_process.DoesNotRun, "build")
        + api.post_process(post_process.StatusSuccess)
        + api.post_process(post_process.DropExpectation)
    )

    yield (
        api.buildbucket_util.test(
            "baking_delay_build_needed",
            bucket="prod",
            git_repo=GIT_URL,
            revision="a" * 40,
        )
        + api.platform.name("linux")
        + api.properties(min_baking_delay_days=14)
        + api.git.get_baked_commit(
            "prepare uploader checkout.get baked commit",
            "b" * 40,
        )
        + api.post_process(
            post_process.MustRun,
            "prepare uploader checkout.checkout baked commit",
        )
        + api.post_process(post_process.MustRun, "build")
        + api.post_process(post_process.StatusSuccess)
        + api.post_process(post_process.DropExpectation)
    )

    yield (
        api.buildbucket_util.test(
            "baking_delay_immature_repo",
            bucket="prod",
            git_repo=GIT_URL,
            revision="a" * 40,
        )
        + api.platform.name("linux")
        + api.properties(min_baking_delay_days=14)
        + api.git.get_baked_commit(
            "prepare uploader checkout.get baked commit",
            "",
            retcode=128,
        )
        + api.post_process(
            post_process.MustRun,
            "prepare uploader checkout.no baked commit found; waiting for repository to mature",
        )
        + api.post_process(post_process.DoesNotRun, "build")
        + api.post_process(post_process.StatusSuccess)
        + api.post_process(post_process.DropExpectation)
    )

    yield (
        api.buildbucket_util.test(
            "baking_delay_force_build",
            bucket="prod",
            git_repo=GIT_URL,
            revision="a" * 40,
        )
        + api.platform.name("linux")
        + api.properties(min_baking_delay_days=14, force_build=True)
        + api.git.get_baked_commit(
            "prepare uploader checkout.get baked commit",
            "b" * 40,
        )
        + api.post_process(
            post_process.MustRun,
            "prepare uploader checkout.checkout baked commit",
        )
        + api.post_process(
            post_process.DoesNotRun,
            "prepare uploader checkout.check package up-to-date",
        )
        + api.post_process(post_process.MustRun, "build")
        + api.post_process(post_process.StatusSuccess)
        + api.post_process(post_process.DropExpectation)
    )

    yield (
        api.buildbucket_util.test(
            "baking_delay_tryjob",
            bucket="try",
            git_repo=GIT_URL,
            revision="a" * 40,
            tryjob=True,
        )
        + api.platform.name("linux")
        + api.properties(min_baking_delay_days=14)
        + api.git.get_baked_commit(
            "prepare uploader checkout.get baked commit",
            "b" * 40,
        )
        + api.post_process(
            post_process.MustRun,
            "prepare uploader checkout.checkout baked commit",
        )
        + api.post_process(
            post_process.DoesNotRun,
            "prepare uploader checkout.check package up-to-date",
        )
        + api.post_process(post_process.MustRun, "build")
        + api.post_process(post_process.StatusSuccess)
        + api.post_process(post_process.DropExpectation)
    )