recipes/contrib/go_toolchain.py

# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building Go toolchain."""

from PB.recipes.fuchsia.contrib.go_toolchain import InputProperties

DEPS = [
    "fuchsia/buildbucket_util",
    "fuchsia/cipd_util",
    "fuchsia/git_checkout",
    "fuchsia/go",
    "fuchsia/macos_sdk",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties

# Files and directories to clean up before and after a build, mirroring what go
# itself does for a release.
# See https://github.com/golang/build/blob/6c34d49dff4864185cb351d06518ce3a76efb6a2/cmd/release/release.go#L240:L248
PREBUILD_CLEAN_FILES = [
    ".gitattributes",
    ".gitignore",
    ".hgignore",
    ".hgtags",
    "misc/dashboard",
    "misc/makerelease",
]
PREBUILD_CLEAN_DIRS = [
    ".git",
    ".github",
]
# See https://github.com/golang/build/blob/6c34d49dff4864185cb351d06518ce3a76efb6a2/cmd/release/release.go#L464:L467
POSTBUILD_CLEAN_FILES = [
    "VERSION.cache",
]
POSTBUILD_CLEAN_DIRS = [
    "pkg/bootstrap",
    "pkg/obj",
]

FUCHSIA_THIRD_PARTY_GO_REPO = "https://fuchsia.googlesource.com/third_party/go"


def RunSteps(api, props):
    host_platform = api.cipd_util.platform_name
    target_platform = props.platform or host_platform

    go_dir, revision = api.git_checkout(
        repo=FUCHSIA_THIRD_PARTY_GO_REPO, rebase_merges=True
    )

    with api.step.nest("ensure_packages"), api.context(infra_steps=True):
        cipd_dir = api.path.start_dir / "cipd"
        pkgs = api.cipd.EnsureFile()
        pkgs.add_package(
            "fuchsia/third_party/clang/${platform}",
            "integration",
        )
        api.cipd.ensure(cipd_dir, pkgs)

    # Create a symlink for `ar` on PATH. Despite Go accepting the environment
    # variable `AR`, it appears some code paths hardcode `ar` and do PATH
    # lookups and do not find `llvm-ar`.
    tool_symlink_dir = api.path.mkdtemp()
    if not api.platform.is_win:
        with api.step.nest("create tool symlinks"):
            for tool in ["ar"]:
                api.file.symlink(
                    f"create {tool}".format(tool),
                    cipd_dir.joinpath("bin", f"llvm-{tool}".format(tool)),
                    tool_symlink_dir / tool,
                )

    cipd_os, cipd_cpu = target_platform.split("-")
    goos = cipd_os.replace("mac", "darwin")
    goarch = cipd_cpu.replace("armv6", "arm")

    # Set CGO_ENABLED=0 as it is not necessary for builds of the Go toolchain
    # post-1.20 and we do not have `gcc` installed for it to find. However,
    # some tests appear to ignore CGO_ENABLED=0 and attempt to run compiles
    # with a cgo toolchain, so feed it `CC` and `AR` to make those happy. Also
    # see additional comments about `ar` above.
    env = {
        "AR": cipd_dir.joinpath("bin", "llvm-ar"),
        "CC": cipd_dir.joinpath("bin", "clang"),
        "CGO_ENABLED": 0,
        "GOOS": goos,
        "GOARCH": goarch,
        "GOROOT_BOOTSTRAP": api.go.go_root,
    }

    with api.macos_sdk(), api.context(
        cwd=go_dir / "src",
        env=env,
        env_prefixes={"PATH": [cipd_dir / "bin", tool_symlink_dir]},
    ):
        with api.step.nest("pre-build file removal"):
            for filename in PREBUILD_CLEAN_FILES:
                api.file.remove(filename, go_dir / filename)
            for dirname in PREBUILD_CLEAN_DIRS:
                api.file.rmtree(dirname, go_dir / dirname)

        api.step("build", [go_dir.joinpath("src", "make.bash")])

        if host_platform == target_platform:
            with api.context(env_prefixes={"PATH": [go_dir / "bin"]}):
                api.step(
                    "test",
                    [
                        go_dir.joinpath("src", "run.bash"),
                    ],
                )

    # Remove after testing, as some tests may rely on the contents.
    with api.step.nest("post-build file removal"):
        for filename in POSTBUILD_CLEAN_FILES:
            api.file.remove(filename, go_dir / filename)
        for dirname in POSTBUILD_CLEAN_DIRS:
            api.file.rmtree(dirname, go_dir / dirname)

    # The builders cross-compile for arm64, which results in the bin/go binary
    # being x86 with the actual arm binary at bin/linux_arm64/go.
    # Modify the output to ensure that bin/go is the appropriate binary.
    if cipd_os == "linux" and cipd_cpu == "arm64":
        with api.step.nest("post-build linux-arm64 modifications"):
            api.file.move(
                "move bin/linux-arm64/go to bin/go",
                go_dir / "bin/linux_arm64/go",
                go_dir / "bin/go",
            )
            api.file.rmtree("remove bin/linux_arm64/", go_dir / "bin/linux_arm64")

    if api.buildbucket_util.is_dev_or_try:
        return

    go_version = (
        api.file.read_text(
            "read go version", go_dir / "VERSION", test_data="go1.8\nblah"
        ).strip()
        # The VERSION file may contain multiple lines, only the first line
        # should be considered as the version.
        .splitlines()[0]
    )
    assert go_version, "Cannot determine Go version"

    api.cipd_util.upload_package(
        "fuchsia/go/" + target_platform,
        go_dir,
        search_tag={"git_revision": revision},
        repository=FUCHSIA_THIRD_PARTY_GO_REPO,
        metadata=[("version", go_version)],
    )


def GenTests(api):
    properties = {
        "checkout.project": "integration",
        "checkout.manifest": "third_party/go",
        "checkout.remote": "https://fuchsia.googlesource.com/third_party/go",
    }
    amd64 = {
        "platform": "linux-amd64",
    }
    arm64 = {
        "platform": "linux-arm64",
    }

    yield api.buildbucket_util.test("cq-amd64", tryjob=True) + api.properties(
        **properties
    ) + api.properties(**amd64)
    yield api.buildbucket_util.test("ci-amd64", tryjob=False) + api.properties(
        **properties
    ) + api.properties(**amd64)
    yield api.buildbucket_util.test("cq-arm64", tryjob=True) + api.properties(
        **properties
    ) + api.properties(**arm64)