recipes/jq.py

# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Recipe for building and uploading jq to CIPD."""


DEPS = [
    "fuchsia/archive",
    "fuchsia/git",
    "fuchsia/status_check",
    "fuchsia/upload",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

JQ_REPO_URL = "https://fuchsia.googlesource.com/third_party/github.com/stedolan/jq"
JQ_REPO_REVISION = "a17dd3248a666d01be75f6b16be37e80e20b0954"

AUTOCONF_PKG = ("fuchsia/third_party/autoconf/${platform}", "version:2.69")
AUTOMAKE_PKG = ("fuchsia/third_party/automake/${platform}", "version:1.16.2")
BISON_PKG = ("fuchsia/third_party/bison/${platform}", "version:3.7")
LIBTOOL_PKG = ("fuchsia/third_party/libtool/${platform}", "version:2.4.6")
M4_PKG = ("fuchsia/third_party/m4/${platform}", "version:1.4.18")
MAKE_PKG = ("fuchsia/third_party/make/${platform}", "version:4.3")

GNU_BUILD_TOOL_PKGS = (
    AUTOCONF_PKG,
    AUTOMAKE_PKG,
    BISON_PKG,
    LIBTOOL_PKG,
    M4_PKG,
    MAKE_PKG,
)

CLANG_PKG = ("fuchsia/third_party/clang/${platform}", "integration")
LINUX_SYSROOT_PKG = (
    "fuchsia/third_party/sysroot/linux",
    "git_revision:c912d089c3d46d8982fdef76a50514cca79b6132",
    "sysroot",
)


def sysroot(api, cipd_dir):
    if api.platform.is_linux:
        return cipd_dir.join(LINUX_SYSROOT_PKG[-1])
    elif api.platform.is_mac:  # pragma: no cover
        # TODO(fxbug.dev/3043): Eventually use our own hermetic sysroot as for Linux.
        step_result = api.step(
            "xcrun",
            ["xcrun", "--sdk", "macosx", "--show-sdk-path"],
            stdout=api.raw_io.output(name="sdk-path", add_output_log=True),
            step_test_data=lambda: api.raw_io.test_api.stream_output(
                "/some/xcode/path"
            ),
        )
        return step_result.stdout.strip()
    raise api.step.StepFailure("unsupported platform")  # pragma: no cover


def RunSteps(api):
    cipd_dir = api.path["start_dir"].join("cipd")
    src_dir = api.path["start_dir"].join("jq")
    with api.context(infra_steps=True):
        to_ensure = [CLANG_PKG] + list(GNU_BUILD_TOOL_PKGS)
        if api.platform.is_linux:
            to_ensure.append(LINUX_SYSROOT_PKG)

        pkgs = api.cipd.EnsureFile()
        for pkg in to_ensure:
            pkgs.add_package(*pkg)
        api.cipd.ensure(cipd_dir, pkgs)

        with api.step.nest("checkout"):
            api.git.checkout(
                JQ_REPO_URL,
                path=src_dir,
                ref=JQ_REPO_REVISION,
                recursive=True,
                submodules=True,
            )

    install_dir = api.path["start_dir"].join("install")
    with api.context(
        cwd=src_dir,
        env_prefixes={"PATH": [cipd_dir.join("bin")]},
        # TODO(fxbug.dev/58251): remove environment variable when possible.
        env={"M4": cipd_dir.join("bin", "m4")},
    ):
        api.step(
            "autoreconf", ["autoreconf", "--verbose", "--install", "--force"],
        )

        configure_flags = ["--sysroot=%s" % sysroot(api, cipd_dir), "-O3"]
        if not api.platform.is_mac:
            # TODO: LTO is currently failing on macOS
            configure_flags.append("-flto")
        configure_variables = {
            "CC": "clang",
            "CXX": "clang++",
            "CFLAGS": " ".join(configure_flags),
            "CPPFLAGS": " ".join(configure_flags),
            "CXXFLAGS": " ".join(configure_flags),
            "LDFLAGS": " ".join(configure_flags),
        }
        if api.platform.is_linux:
            configure_variables.update(
                {
                    "AR": "llvm-ar",
                    "RANLIB": "llvm-ranlib",
                    "NM": "llvm-nm",
                    "STRIP": "llvm-strip",
                    "OBJCOPY": "llvm-objcopy",
                }
            )

        api.step(
            "configure",
            [
                "./configure",
                "--with-oniguruma=builtin",
                "--prefix=",
                "--disable-dependency-tracking",
                "--disable-shared",
                "--enable-all-static",
            ]
            + sorted(["%s=%s" % (k, v) for k, v in configure_variables.iteritems()]),
        )

        api.step("make", ["make", "-j%d" % api.platform.cpu_count])
        api.step("make check", ["make", "check"])
        api.step(
            "make install", ["make", "install", "DESTDIR=%s" % install_dir],
        )

    if api.buildbucket.builder_id.bucket == "ci":
        api.archive.upload(install_dir, output_property="isolated")
    if api.buildbucket.builder_id.bucket == "prod":
        # TODO: factor this out into a host_build recipe module.
        platform = "%s-%s" % (
            api.platform.name.replace("win", "windows"),
            {"intel": {32: "386", 64: "amd64",}, "arm": {32: "armv6", 64: "arm64",},}[
                api.platform.arch
            ][api.platform.bits],
        )
        api.upload.cipd_package(
            "fuchsia/third_party/jq/%s" % platform,
            install_dir,
            [api.upload.DirectoryPath(install_dir)],
            {"git_revision": JQ_REPO_REVISION},
        )


def GenTests(api):
    yield api.status_check.test("ci") + api.buildbucket.ci_build(
        project="fuchsia", bucket="ci",
    ) + api.platform.name("linux")
    yield api.status_check.test("prod") + api.buildbucket.ci_build(
        project="fuchsia", bucket="prod",
    ) + api.platform.name("mac")