| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed under the Apache License, Version 2.0 |
| # that can be found in the LICENSE file. |
| """Recipe for building Ninja.""" |
| |
| from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest |
| from PB.recipes.fuchsia.contrib.ninja import InputProperties |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/cas_util", |
| "fuchsia/cipd_util", |
| "fuchsia/git_checkout", |
| "fuchsia/macos_sdk", |
| "fuchsia/platform_util", |
| "fuchsia/windows_sdk", |
| "recipe_engine/buildbucket", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/platform", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| GIT_URL = "https://fuchsia.googlesource.com/third_party/github.com/ninja-build/ninja" |
| |
| # Leave GIT_REVISION empty to use the latest buildbucket input |
| GIT_REVISION = None |
| |
| CIPD_SERVER_HOST = "chrome-infra-packages.appspot.com" |
| |
| # On select platforms, link the Ninja executable against rpmalloc for a small 10% speed boost. |
| RPMALLOC_GIT_URL = ( |
| "https://fuchsia.googlesource.com/third_party/github.com/mjansson/rpmalloc" |
| ) |
| RPMALLOC_REVISION = "b097fd0916500439721a114bb9cd8d14bd998683" |
| |
| # Used to convert os and arch strings to rpmalloc format |
| RPMALLOC_MAP = { |
| "intel": "x86-64", |
| "amd64": "x86-64", |
| "arm": "arm64", |
| "mac": "macos", |
| } |
| |
| |
| def slashes(api, path): |
| # CMake only accept '/' as path delimiter. This helper function replaces '\' |
| # with '/' on Windows platform. |
| return path.replace("\\", "/") if api.platform.is_win else path |
| |
| |
| def read_revision(api, source): |
| for curline in source.splitlines(): |
| if "kNinjaVersion" in curline: |
| revision = curline[curline.find("=") + 1 : curline.find(";")].strip(' "') |
| return revision |
| raise api.step.StepFailure("revision string not found") # pragma no cover |
| |
| |
| def _get_sysroot(api, cipd_dir, platform): |
| if platform.is_linux: |
| return cipd_dir.join("sysroot") |
| if platform.is_mac: |
| step_result = api.step( |
| "xcrun", |
| ["xcrun", "--sdk", "macosx", "--show-sdk-path"], |
| stdout=api.raw_io.output_text(name="sdk-path", add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text( |
| "/some/xcode/path" |
| ), |
| ) |
| return step_result.stdout.strip() |
| raise api.step.StepFailure("unsupported platform") # pragma: no cover |
| |
| |
| def _get_build_config(api, cipd_dir, platform): |
| config = {} |
| if platform.is_win: |
| config.update( |
| { |
| "cc": cipd_dir.join("bin", "clang-cl.exe"), |
| "cxx": cipd_dir.join("bin", "clang-cl.exe"), |
| "ninja": cipd_dir.join("ninja.exe"), |
| "sysroot": api.windows_sdk.sdk_dir, |
| } |
| ) |
| else: |
| config.update( |
| { |
| "cc": cipd_dir.join("bin", "clang"), |
| "cxx": cipd_dir.join("bin", "clang++"), |
| "ninja": cipd_dir.join("ninja"), |
| "sysroot": _get_sysroot(api, cipd_dir, platform), |
| } |
| ) |
| if platform.is_mac: |
| config["link_flags"] = "-nostdlib++ %s" % cipd_dir.join("lib", "libc++.a") |
| |
| return config |
| |
| |
| def RunSteps(api, props): |
| host_platform = api.platform_util.host |
| target_platform = ( |
| api.platform_util.platform(props.platform) if props.platform else host_platform |
| ) |
| |
| manifest = Manifest() |
| |
| if not GIT_REVISION: |
| src_dir, revision = api.git_checkout(GIT_URL) |
| else: # pragma: nocover |
| src_dir, revision = api.git_checkout(GIT_URL, revision=GIT_REVISION) |
| |
| git_checkout = manifest.directories[str(src_dir)].git_checkout |
| git_checkout.repo_url = GIT_URL |
| git_checkout.revision = revision |
| version_data = api.file.read_text( |
| "read ninja revision", |
| src_dir.join("src", "version.cc"), |
| test_data='\nconst char* kNinjaVersion = "1.10.2.git";\n', |
| ) |
| ninja_version = read_revision(api, version_data) |
| api.step.empty("ninja version", step_text=ninja_version) |
| |
| with api.step.nest("ensure packages"), api.context(infra_steps=True): |
| cipd_dir = api.path["start_dir"].join("cipd") |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration") |
| if api.platform.is_linux: |
| pkgs.add_package( |
| "fuchsia/third_party/sysroot/linux", |
| "git_revision:47910c0625ad625def7d9e21c9213c91eb9cfa51", |
| "sysroot", |
| ) |
| pkgs.add_package( |
| "fuchsia/third_party/cmake/${platform}", |
| "integration", |
| ) |
| pkgs.add_package( |
| "fuchsia/third_party/ninja/${platform}", |
| "integration", |
| ) |
| ensured = api.cipd.ensure(cipd_dir, pkgs) |
| for subdir, pins in ensured.items(): |
| directory = manifest.directories[str(cipd_dir.join(subdir))] |
| directory.cipd_server_host = CIPD_SERVER_HOST |
| for pin in pins: |
| directory.cipd_package[pin.package].instance_id = pin.instance_id |
| |
| api.file.write_proto( |
| "source manifest", src_dir.join("source_manifest.json"), manifest, "JSONPB" |
| ) |
| |
| use_rpmalloc = api.platform.is_linux or api.platform.is_mac |
| if use_rpmalloc: |
| with api.step.nest("rpmalloc"): |
| rpmalloc_src_dir, _ = api.git_checkout( |
| RPMALLOC_GIT_URL, fallback_ref=RPMALLOC_REVISION |
| ) |
| |
| # Patch configure.py to add -Wno-ignored-optimization-flag since |
| # Clang will complain when `-funit-at-a-time`is being used because |
| # it is now obsolete for this compiler. |
| build_ninja_clang_path = api.path.join( |
| rpmalloc_src_dir, "build/ninja/clang.py" |
| ) |
| build_ninja_clang_py = api.file.read_text( |
| "read %s" % build_ninja_clang_path, |
| build_ninja_clang_path, |
| "CXXFLAGS = ['-Wno-disabled-macro-expansion']", |
| ) |
| build_ninja_clang_py = build_ninja_clang_py.replace( |
| "'-Wno-disabled-macro-expansion'", |
| "'-Wno-disabled-macro-expansion', '-Wno-ignored-optimization-argument'", |
| ) |
| api.file.write_text( |
| "write %s" % build_ninja_clang_path, |
| build_ninja_clang_path, |
| build_ninja_clang_py, |
| ) |
| |
| with api.macos_sdk(), api.windows_sdk(): |
| rpmalloc_os, rpmalloc_arch = api.platform.name, api.platform.arch |
| rpmalloc_os = RPMALLOC_MAP.get(rpmalloc_os, rpmalloc_os) |
| rpmalloc_arch = RPMALLOC_MAP.get(rpmalloc_arch, rpmalloc_arch) |
| |
| config = _get_build_config(api, cipd_dir, target_platform) |
| triple = "--target=%s" % target_platform.triple |
| env = { |
| "CC": config["cc"], |
| "CXX": config["cxx"], |
| "AR": "%s" % cipd_dir.join("bin", "llvm-ar"), |
| "CFLAGS": "%s --sysroot=%s" % (triple, config["sysroot"]), |
| } |
| env["LDFLAGS"] = triple |
| if "link_flags" in config: |
| env["LDFLAGS"] += " %s" % config["link_flags"] |
| |
| with api.step.nest("build rpmalloc-" + api.platform.name), api.context( |
| env=env, cwd=rpmalloc_src_dir |
| ): |
| api.step( |
| "configure", |
| [ |
| "python", |
| rpmalloc_src_dir.join("configure.py"), |
| "-c", |
| "release", |
| "-a", |
| rpmalloc_arch, |
| "--lto", |
| ], |
| ) |
| |
| # NOTE: Only build the static library. |
| if rpmalloc_os == "macos": |
| # For MacOS, the library path does not include the architecture, |
| # Probably because it can contain multi-arch binaries. |
| rpmalloc_static_lib = api.path.join( |
| "lib", |
| rpmalloc_os, |
| "release", |
| "librpmallocwrap.a", |
| ) |
| else: |
| rpmalloc_static_lib = api.path.join( |
| "lib", |
| rpmalloc_os, |
| "release", |
| rpmalloc_arch, |
| "librpmallocwrap.a", |
| ) |
| api.step("ninja", [cipd_dir.join("ninja"), rpmalloc_static_lib]) |
| |
| staging_dir = api.path.mkdtemp("ninja") |
| build_dir = staging_dir.join("ninja_build_dir") |
| api.file.ensure_directory("create build dir", build_dir) |
| pkg_dir = staging_dir.join("ninja") |
| api.file.ensure_directory("create pkg dir", pkg_dir) |
| |
| cmake_src_dir = src_dir |
| |
| if use_rpmalloc: |
| # Building Ninja with rpmalloc without modifying the upstream project. |
| # See https://github.com/ninja-build/ninja/pull/2071 |
| # In a nutshell, generate a tiny CMakeList.txt file in a different directory |
| cmake_src_dir = staging_dir.join("rpmalloc_link_src_dir") |
| api.file.ensure_directory("create rpmalloc_link source dir", cmake_src_dir) |
| cmakelists_txt_file = cmake_src_dir.join("CMakeLists.txt") |
| cmake_ninja_src_dir = api.path.relpath(src_dir, cmake_src_dir) |
| cmake_rpmalloc_lib = api.path.abspath( |
| api.path.join(rpmalloc_src_dir, rpmalloc_static_lib) |
| ) |
| api.file.write_text( |
| "generate rpmalloc_link CMakeLists.txt", |
| cmakelists_txt_file, |
| """cmake_minimum_required(VERSION 3.15) |
| project(ninja-rpmalloc) |
| include(CTest) |
| add_subdirectory("%s" ninja) |
| target_link_libraries(ninja PRIVATE "%s" -lpthread -ldl) |
| """ |
| % (cmake_ninja_src_dir, cmake_rpmalloc_lib), |
| ) |
| |
| arguments = {} |
| with api.macos_sdk(), api.windows_sdk(), api.context(cwd=build_dir): |
| arguments = _get_build_config(api, cipd_dir, target_platform) |
| options = [ |
| "-GNinja", |
| "-DCMAKE_C_COMPILER={cc}", |
| "-DCMAKE_CXX_COMPILER={cxx}", |
| "-DCMAKE_ASM_COMPILER={cc}", |
| "-DCMAKE_MAKE_PROGRAM={ninja}", |
| "-DCMAKE_BUILD_TYPE=Release", |
| "-DCMAKE_INSTALL_PREFIX=", |
| "-DCMAKE_SYSROOT={sysroot}", |
| ] |
| |
| if target_platform != host_platform: |
| options.extend( |
| [ |
| "-DCMAKE_C_COMPILER_TARGET=%s" % target_platform.triple, |
| "-DCMAKE_CXX_COMPILER_TARGET=%s" % target_platform.triple, |
| "-DCMAKE_ASM_COMPILER_TARGET=%s" % target_platform.triple, |
| "-DCMAKE_SYSTEM_NAME=%s" |
| % target_platform.os.replace("mac", "darwin").title(), |
| ] |
| ) |
| |
| if api.platform.is_linux: |
| options.extend( |
| [ |
| # TODO(phosek): see https://github.com/ninja-build/ninja/issues/1821 |
| "-DCMAKE_C_FLAGS=-DUSE_PPOLL", |
| "-DCMAKE_CXX_FLAGS=-DUSE_PPOLL", |
| ] |
| ) |
| |
| if "link_flags" in arguments: |
| options.extend( |
| [ |
| "-DCMAKE_%s_LINKER_FLAGS={link_flags}" % mode |
| for mode in ["SHARED", "MODULE", "EXE"] |
| ] |
| ) |
| |
| api.step( |
| "configure", |
| [cipd_dir.join("bin", "cmake")] |
| + [slashes(api, option.format(**arguments)) for option in options] |
| + [cmake_src_dir], |
| ) |
| api.step("build", [cipd_dir.join("ninja")]) |
| if target_platform == host_platform: |
| api.step("test", [cipd_dir.join("ninja"), "test"]) |
| |
| with api.context(env={"DESTDIR": pkg_dir}): |
| if api.platform.is_win: |
| api.step("install", [cipd_dir.join("ninja"), "install"]) |
| else: |
| api.step("install", [cipd_dir.join("ninja"), "install/strip"]) |
| |
| ninja = pkg_dir.join("bin", "ninja" + (".exe" if api.platform.is_win else "")) |
| |
| # TODO(fxbug.dev/86997): Remove the duplicated ninja binary after transition completes |
| copied_ninja = pkg_dir.join("ninja" + (".exe" if api.platform.is_win else "")) |
| api.file.copy("duplicate ninja binary", ninja, copied_ninja) |
| |
| # Upload the installation to CAS. |
| api.cas_util.upload(pkg_dir, [ninja, copied_ninja], output_property="isolated") |
| |
| if api.buildbucket.builder_id.bucket == "prod": |
| # Upload the installation to CIPD for production builds. |
| api.cipd_util.upload_package( |
| "fuchsia/third_party/ninja/%s" % target_platform, |
| pkg_dir, |
| pkg_paths=[ninja, copied_ninja], |
| search_tag={"git_revision": revision}, |
| repository=GIT_URL, |
| metadata=[("version", ninja_version)], |
| ) |
| |
| |
| def GenTests(api): |
| for platform in ("linux", "mac", "win"): |
| yield ( |
| api.buildbucket_util.test( |
| platform, bucket="prod", git_repo=GIT_URL, revision="a" * 40 |
| ) |
| + api.platform.name(platform) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "mac-arm64", bucket="prod", git_repo=GIT_URL, revision="a" * 40 |
| ) |
| + api.platform.name("mac") |
| + api.properties(platform="mac-arm64") |
| ) |
