recipes/contrib/ninja.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
"""Recipe for building Ninja."""

from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest
from PB.recipes.fuchsia.contrib.ninja import InputProperties

DEPS = [
    "fuchsia/buildbucket_util",
    "fuchsia/cas_util",
    "fuchsia/cipd_util",
    "fuchsia/cmake",
    "fuchsia/git_checkout",
    "fuchsia/macos_sdk",
    "fuchsia/ninja",
    "fuchsia/platform_util",
    "fuchsia/windows_sdk",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties

GIT_URL = "https://fuchsia.googlesource.com/third_party/github.com/ninja-build/ninja"
GIT_BRANCH = "main"

CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com"

GOOGLETEST_GIT_URL = (
    "https://fuchsia.googlesource.com/third_party/github.com/google/googletest"
)
GOOGLETEST_REVISION = "refs/tags/v1.14.0"

# On select platforms, link the Ninja executable against rpmalloc for a small 10% speed boost.
RPMALLOC_GIT_URL = (
    "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc"
)
RPMALLOC_REVISION = "refs/tags/1.4.4"

# Used to convert os and arch strings to rpmalloc format
RPMALLOC_MAP = {
    "intel": "x86-64",
    "amd64": "x86-64",
    "arm": "arm64",
    "mac": "macos",
}


def RunSteps(api, props):
    host_platform = api.platform_util.host
    target_platform = (
        api.platform_util.platform(props.platform) if props.platform else host_platform
    )

    manifest = Manifest()
    src_dir, revision = api.git_checkout(GIT_URL, fallback_ref=GIT_BRANCH)
    git_checkout = manifest.directories[str(src_dir)].git_checkout
    git_checkout.repo_url = GIT_URL
    git_checkout.revision = revision
    version_data = api.file.read_text(
        "read ninja revision",
        src_dir.joinpath("src", "version.cc"),
        test_data='\nconst char* kNinjaVersion = "1.10.2.git";\n',
    )
    ninja_version = read_revision(api, version_data)
    api.step.empty("ninja version", step_text=ninja_version)

    with api.step.nest("ensure packages"), api.context(infra_steps=True):
        cipd_dir = api.path.start_dir / "cipd"
        pkgs = api.cipd.EnsureFile()
        pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration")
        if api.platform.is_linux:
            pkgs.add_package(
                "fuchsia/third_party/sysroot/linux",
                "integration",
                "sysroot",
            )
        ensured = api.cipd.ensure(cipd_dir, pkgs)
        for subdir, pins in ensured.items():
            directory = manifest.directories[str(cipd_dir / subdir)]
            directory.cipd_server_host = CIPD_SERVER_HOST
            for pin in pins:
                directory.cipd_package[pin.package].instance_id = pin.instance_id

    api.file.write_proto(
        "source manifest",
        src_dir / "source_manifest.json",
        manifest,
        "JSONPB",
    )

    with api.step.nest("googletest"):
        googletest_src_dir, _ = api.git_checkout(
            GOOGLETEST_GIT_URL, fallback_ref=GOOGLETEST_REVISION
        )
        googletest_install_dir = api.path.start_dir / "googletest_install"

        arguments = {}
        with api.macos_sdk(), api.windows_sdk():
            arguments = get_build_config(api, cipd_dir, target_platform)
            options = get_cmake_options(target_platform, host_platform)

            # Ensure everything is installed directly under the installation
            # directory, and not DESTDIR/usr/local on Unix and
            # c;/Program Files/${PROJECT_NAME} on Windows.
            options.append(f"-DCMAKE_INSTALL_PREFIX={googletest_install_dir}")

            api.cmake.build_with_ninja(
                src_dir=googletest_src_dir,
                build_type="Release",
                extra_args=[option.format(**arguments) for option in options],
            )

    use_rpmalloc = api.platform.is_linux or api.platform.is_mac
    if use_rpmalloc:
        with api.step.nest("rpmalloc"):
            rpmalloc_src_dir, _ = api.git_checkout(
                RPMALLOC_GIT_URL, fallback_ref=RPMALLOC_REVISION
            )

            # Patch configure.py to replace -Werror with -Wno-error because
            # it is causing too many compilation issues when building rpmalloc
            # with this recipe.
            build_ninja_clang_path = api.path.join(
                rpmalloc_src_dir, "build", "ninja", "clang.py"
            )
            build_ninja_clang_py = api.file.read_text(
                f"read {build_ninja_clang_path}",
                build_ninja_clang_path,
                "CXXFLAGS = ['-Wno-disabled-macro-expansion']",
            )
            build_ninja_clang_py = build_ninja_clang_py.replace(
                "'-Werror',", "'-Wno-error',"
            )
            api.file.write_text(
                f"write {build_ninja_clang_path}",
                build_ninja_clang_path,
                build_ninja_clang_py,
            )

            with api.macos_sdk(), api.windows_sdk():
                rpmalloc_os, rpmalloc_arch = api.platform.name, api.platform.arch
                rpmalloc_os = RPMALLOC_MAP.get(rpmalloc_os, rpmalloc_os)
                rpmalloc_arch = RPMALLOC_MAP.get(rpmalloc_arch, rpmalloc_arch)

                config = get_build_config(api, cipd_dir, target_platform)
                triple = f"--target={target_platform.triple}"
                env = {
                    "CC": config["cc"],
                    "CXX": config["cxx"],
                    "AR": f"{cipd_dir.joinpath('bin', 'llvm-ar')}",
                    "CFLAGS": f"{triple} --sysroot={config['sysroot']}",
                }
                env["LDFLAGS"] = triple
                if "link_flags" in config:
                    env["LDFLAGS"] += f" {config['link_flags']}"

                with api.step.nest(f"build rpmalloc-{api.platform.name}"), api.context(
                    env=env, cwd=rpmalloc_src_dir
                ):
                    api.step(
                        "configure",
                        [
                            "python3",
                            rpmalloc_src_dir / "configure.py",
                            "-c",
                            "release",
                            "-a",
                            rpmalloc_arch,
                            "--lto",
                        ],
                    )

                    # NOTE: Only build the static library.
                    if rpmalloc_os == "macos":
                        # For MacOS, the library path does not include the architecture,
                        # Probably because it can contain multi-arch binaries.
                        rpmalloc_static_lib = api.path.join(
                            "lib",
                            rpmalloc_os,
                            "release",
                            "librpmallocwrap.a",
                        )
                    else:
                        rpmalloc_static_lib = api.path.join(
                            "lib",
                            rpmalloc_os,
                            "release",
                            rpmalloc_arch,
                            "librpmallocwrap.a",
                        )
                    api.ninja("ninja", [rpmalloc_static_lib])

    staging_dir = api.path.mkdtemp("ninja")
    build_dir = staging_dir / "ninja_build_dir"
    api.file.ensure_directory("create build dir", build_dir)
    pkg_dir = staging_dir / "ninja"
    api.file.ensure_directory("create pkg dir", pkg_dir)
    cmake_src_dir = src_dir

    if use_rpmalloc:
        # Building Ninja with rpmalloc without modifying the upstream project.
        # See https://github.com/ninja-build/ninja/pull/2071
        # In a nutshell, generate a tiny CMakeList.txt file in a different directory
        cmake_src_dir = staging_dir / "rpmalloc_link_src_dir"
        api.file.ensure_directory("create rpmalloc_link source dir", cmake_src_dir)
        cmakelists_txt_file = cmake_src_dir / "CMakeLists.txt"
        cmake_ninja_src_dir = api.path.relpath(src_dir, cmake_src_dir)
        cmake_rpmalloc_lib = api.path.abspath(
            api.path.join(rpmalloc_src_dir, rpmalloc_static_lib)
        )
        api.file.write_text(
            "generate rpmalloc_link CMakeLists.txt",
            cmakelists_txt_file,
            f"""cmake_minimum_required(VERSION 3.15)
project(ninja-rpmalloc)
include(CTest)
add_subdirectory("{cmake_ninja_src_dir}" ninja)
target_link_libraries(ninja PRIVATE "{cmake_rpmalloc_lib}" -lpthread -ldl)
""",
        )

    arguments = {}
    with api.macos_sdk(), api.windows_sdk():
        arguments = get_build_config(api, cipd_dir, target_platform)
        options = get_cmake_options(target_platform, host_platform)

        if "link_flags" in arguments:
            options.extend(
                [
                    "-DCMAKE_%s_LINKER_FLAGS={link_flags}" % mode
                    for mode in ["SHARED", "MODULE", "EXE"]
                ]
            )

        # Ensure that `ninja test` invoked below prints usable output to
        # stdout. Using --verbose is preferred to --output-on-failure
        # to verify the list of unit tests that were really run.
        options.append("-DCMAKE_CTEST_ARGUMENTS=--verbose")

        # Ensure local GoogleTest sources are used. otherwise the CMakeList.txt
        # file will download an archive from the network.
        options.append(f"-DGTEST_ROOT={googletest_install_dir}")

        api.cmake.build_with_ninja(
            src_dir=cmake_src_dir,
            build_dir=build_dir,
            build_type="Release",
            extra_args=[option.format(**arguments) for option in options],
            install_cmd="install" if api.platform.is_win else "install/strip",
            install_dir=pkg_dir,
        )
        if target_platform == host_platform:
            api.ninja("test", ["test"], build_dir=build_dir)

    ninja = pkg_dir.joinpath("bin", f"ninja{'.exe' if api.platform.is_win else ''}")

    # TODO(fxbug.dev/86997): Remove the duplicated ninja binary after transition completes
    copied_ninja = pkg_dir.joinpath("ninja" + (".exe" if api.platform.is_win else ""))
    api.file.copy("duplicate ninja binary", ninja, copied_ninja)

    # Upload the installation to CAS.
    api.cas_util.upload(pkg_dir, [ninja, copied_ninja], output_property="isolated")

    if api.buildbucket.build.builder.bucket == "prod":
        # Upload the installation to CIPD for production builds.
        api.cipd_util.upload_package(
            f"fuchsia/third_party/ninja/{target_platform}",
            pkg_dir,
            pkg_paths=[ninja, copied_ninja],
            search_tag={"git_revision": revision},
            repository=GIT_URL,
            metadata=[("version", ninja_version)],
        )


def read_revision(api, source):
    for curline in source.splitlines():
        if "kNinjaVersion" in curline:
            revision = curline[curline.find("=") + 1 : curline.find(";")].strip(' "')
            return revision
    raise api.step.StepFailure("revision string not found")  # pragma no cover


def get_sysroot(api, cipd_dir, platform):
    if platform.is_linux:
        return cipd_dir / "sysroot"
    if platform.is_mac:
        return api.macos_sdk.sysroot
    raise api.step.StepFailure("unsupported platform")  # pragma: no cover


def get_build_config(api, cipd_dir, platform):
    config = {
        "ninja": api.ninja.path,
    }
    if platform.is_win:
        config.update(
            {
                "cc": cipd_dir.joinpath("bin", "clang-cl.exe"),
                "cxx": cipd_dir.joinpath("bin", "clang-cl.exe"),
                "sysroot": api.windows_sdk.sdk_dir,
            }
        )
    else:
        config.update(
            {
                "cc": cipd_dir.joinpath("bin", "clang"),
                "cxx": cipd_dir.joinpath("bin", "clang++"),
                "sysroot": get_sysroot(api, cipd_dir, platform),
            }
        )
    if platform.is_mac:
        config["link_flags"] = f"-nostdlib++ {cipd_dir.joinpath('lib', 'libc++.a')}"

    return config


def get_cmake_options(target_platform, host_platform):
    options = [
        "-DCMAKE_C_COMPILER={cc}",
        "-DCMAKE_CXX_COMPILER={cxx}",
        "-DCMAKE_ASM_COMPILER={cc}",
        "-DCMAKE_MAKE_PROGRAM={ninja}",
        "-DCMAKE_INSTALL_PREFIX=",
        "-DCMAKE_SYSROOT={sysroot}",
    ]

    if target_platform != host_platform:
        options.extend(
            [
                f"-DCMAKE_C_COMPILER_TARGET={target_platform.triple}",
                f"-DCMAKE_CXX_COMPILER_TARGET={target_platform.triple}",
                f"-DCMAKE_ASM_COMPILER_TARGET={target_platform.triple}",
                f"-DCMAKE_SYSTEM_NAME={target_platform.os.replace('mac', 'darwin').title()}",
            ]
        )

    return options


def GenTests(api):
    for platform in ("linux", "mac", "win"):
        yield (
            api.buildbucket_util.test(
                platform, bucket="prod", git_repo=GIT_URL, revision="a" * 40
            )
            + api.platform.name(platform)
        )

    yield (
        api.buildbucket_util.test(
            "mac-arm64", bucket="prod", git_repo=GIT_URL, revision="a" * 40
        )
        + api.platform.name("mac")
        + api.properties(platform="mac-arm64")
    )