recipes/goma_toolchain.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building goma toolchain images."""

import re

from recipe_engine.config import List
from recipe_engine.recipe_api import Property

DEPS = [
    "fuchsia/docker",
    "fuchsia/gcloud",
    "fuchsia/gsutil",
    "fuchsia/status_check",
    "recipe_engine/archive",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/python",
    "recipe_engine/step",
    "recipe_engine/time",
    "recipe_engine/url",
]

PROPERTIES = {
    "package": Property(
        kind=str,
        help="cipd path of the toolchain package",
        default="fuchsia/third_party/clang",
    ),
    "version": Property(
        kind=str, help="CIPD version of the toolchain package", default=None
    ),
    "toolchain": Property(kind=str, help="type of the toolchain", default="clang"),
    "platforms": Property(
        kind=List(str),
        help="toolchain host platform",
        default=["linux-amd64", "mac-amd64"],
    ),
    "domain": Property(
        kind=str, help="domain name of container registry", default="gcr.io"
    ),
    "project": Property(
        kind=str, help="name of goma cloud project", default="goma-fuchsia"
    ),
    "root": Property(
        kind=str,
        help="root directory name for container registry",
        default="fuchsia_linux",
    ),
    "update_goma_base_images": Property(
        kind=bool,
        help="Update goma base images even if they already exist",
        default=False,
    ),
}

GOMATOOLS_VERSION = "git_revision:22684ee9174606073a2604e1d58fbf03ba3ad564"

# "A tag name must be valid ASCII and may contain lowercase and uppercase
# letters, digits, underscores, periods and dashes."  So sayeth
# https://docs.docker.com/engine/reference/commandline/tag/
DOCKER_TAG_DISALLOWED_RE = re.compile(r"[^-._a-zA-Z0-9]")


def ensure_gomatools(api, gomatools_dir):
    with api.step.nest("ensure gomatools"):
        pkgs = api.cipd.EnsureFile()
        pkgs.add_package(
            "fuchsia_internal/third_party/goma/server/gomatools/linux-amd64",
            GOMATOOLS_VERSION,
        )
        api.cipd.ensure(gomatools_dir, pkgs)


def generate_time_stamp(api):
    return "{:%Y%m%d_%H%M%S}".format(api.time.utcnow())


def build_goma_base_image(api, dockerfile, image, timestamp=None):
    api.file.copy(
        "copy dockerfile",
        api.resource(dockerfile),
        api.path["cleanup"].join(dockerfile),
    )
    tags = [image]
    if timestamp:
        tags.append(image + ":" + timestamp)
    with api.context(cwd=api.path["cleanup"]):
        api.docker.build(
            dockerfile=api.path["cleanup"].join(dockerfile), tags=tags,
        )
    api.docker("push", image)


def ensure_goma_base_images(
    api, domain, project_id, root, timestamp=None, update_if_exist=False
):
    """Ensure the base goma images in container registry.

    E.g. ensure_goma_base_images(api,
                                 'gcr.io',\
                                 'goma-fuchsia',\
                                 'fuchsia_linux')
        will create docker images at
        gcr.io/goma-fuchsia/fuchsia_linux/base,
        gcr.io/goma-fuchsia/fuchsia_linux/runtime,
        gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform,
        using 'debian:buster-slim' as bottom layer image.

    Args:
        * domain (str) - The domain name of the container register.
        * project (str) - The gcloud project ID.
        * root (str) - The name of container registry root.
        * update_if_exist (str) - Update the container if it exists.
    """
    with api.step.nest("ensure goma base images"):
        # Build goma base image.
        goma_base_image = api.url.join(domain, project_id, root, "base")
        if not api.gcloud.container_image_exists(goma_base_image) or update_if_exist:
            build_goma_base_image(
                api, "Dockerfile_goma_base", goma_base_image, timestamp
            )
        # Build goma runtime image.
        runtime_image = api.url.join(domain, project_id, root, "runtime")
        if not api.gcloud.container_image_exists(runtime_image) or update_if_exist:
            api.docker("pull", goma_base_image)
            api.docker("tag", goma_base_image, "base")
            build_goma_base_image(
                api, "Dockerfile_goma_runtime", runtime_image, timestamp
            )
        # pull-toolchain-runtime
        api.docker("pull", runtime_image)
        api.docker("tag", runtime_image, "toolchain/runtime")
        # Build goma remoteexec-platform
        remote_image = api.url.join(domain, project_id, root, "remoteexec-platform")
        if not api.gcloud.container_image_exists(remote_image) or update_if_exist:
            build_goma_base_image(
                api, "Dockerfile_goma_remoteexec_platform", remote_image, timestamp
            )


def push_toolchain_to_goma(
    api,
    goma_temp_dir,
    project_id,
    root,
    toolchain_tag,
    container_tag,
    container_version="latest",
    stamp=None,
):
    """Push and register the toolchain docker image with Goma.

    E.g. push_toolchain_to_goma(goma_temp_dir,\
                                'goma-fuchsia',\
                                'fuchsia_linux',\
                                'clang-some_hash-linux-amd64',\
                                'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash-linux-amd64',\
                                'latest',
                                '20190909_122500')
        will copy the toolchain binaries from docker image
        'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash-linux-amd64:latest'
        to appropriate gs bucket of the Goma gcloud project and
        register it with Goma server.

    Args:
      * goma_temp_dir (str) - The temp directory for Goma resources.
      * project_id (str) - The gcloud project ID.
      * root (str) - The root path of all toolchains. E.g. 'fuchsia_linux'.
      * toolchain_tag (str) - Toolchain version and platform tag.
      * container_tag (str) - Toolchain container tag.
      * container_version (str) - The container image version. 'latest' by default.
      * stamp (str) - Time stamp string.
    """
    with api.step.nest("push toolchain to goma"):
        # URI = ${domain}/${project_id}/${root}/${toolchain}-${toolchain_version}:${container_version}
        # e.g. URI = gcr.io/goma-fuchsia/fuchsia_linux/clang-0a5975a5bcd59cb07efdc86b47498fbb7c095ab:latest
        if stamp is None:
            stamp = generate_time_stamp(api)  # pragma: no cover
        # extract-goma-files
        with api.docker.create(container_tag + ":" + container_version) as temp_copy:
            api.docker.copy(
                temp_copy, ["/var/cache/goma", "/etc/goma/descriptors"], goma_temp_dir
            )
        api.file.move(
            "rename cmdstorage",
            goma_temp_dir.join("goma", "cmdstorage"),
            goma_temp_dir.join("goma", "sha256"),
        )
        api.file.remove(
            "remove dockerfile if it exists",
            goma_temp_dir.join("Dockerfile_fuchsia_toolchain_push_test"),
        )
        api.file.copy(
            "copy dockerfile",
            api.resource("Dockerfile_fuchsia_toolchain_push_test"),
            goma_temp_dir,
        )
        # build-test-image
        local_toolchain_tag = api.url.join(root, toolchain_tag)
        # Set env
        with api.context(cwd=goma_temp_dir):
            api.docker.build(
                dockerfile=goma_temp_dir.join("Dockerfile_fuchsia_toolchain_push_test"),
                tags=["%s_test" % container_tag, "%s_test" % local_toolchain_tag],
            )
        # run test
        if not local_toolchain_tag[local_toolchain_tag.find("/") + 1 :].startswith(
            "gcc"
        ):
            # TODO(haowei): Fix the goma backend so we can run the
            # toolchain test for gcc.
            api.docker("run", "%s_test" % local_toolchain_tag)

        # store-files
        api.gsutil.upload(
            bucket="%s-files" % project_id,
            src=goma_temp_dir.join("goma", "sha256"),
            dst="",
            recursive=True,
            multithreaded=True,
        )

        api.gsutil.upload(
            bucket="%s-toolchain-config" % project_id,
            src=goma_temp_dir.join("descriptors"),
            dst="%s/%s/" % (root, toolchain_tag),
            recursive=True,
            multithreaded=True,
        )

        api.file.remove("remove seq file", goma_temp_dir.join("seq"))
        api.file.write_text("write seq file", goma_temp_dir.join("seq"), stamp)

        api.gsutil.upload(
            bucket="%s-toolchain-config" % project_id,
            src=goma_temp_dir.join("seq"),
            dst="%s/seq" % root,
            recursive=False,
            multithreaded=True,
        )


def build_and_push_goma_toolchain(
    api, toolchain, version, domain, project, root, platform, goma_temp_dir, timestamp
):
    toolchain_tag = "%s-%s-%s" % (toolchain, version, platform)
    toolchain_tag = DOCKER_TAG_DISALLOWED_RE.sub("_", toolchain_tag)
    container_tag = api.url.join(domain, project, root, toolchain_tag)
    with api.step.nest("copy goma docker configurations"):
        dockerfile_platform = platform.replace("-", "_")
        dockerfile = "Dockerfile_fuchsia_toolchain_{}".format(dockerfile_platform)
        api.file.copy(
            "copy dockerfile", api.resource(dockerfile), goma_temp_dir.join(dockerfile)
        )
        setupfile = "setup_fuchsia_{}_{}".format(toolchain, dockerfile_platform)
        # On CI bots, file.copy complains error 13 if the setup file already
        # exists. Remove it first to avoid this issue.
        api.file.remove("remove existing setup file", goma_temp_dir.join("setup"))
        api.file.copy(
            "copy toolchain setup file",
            api.resource(setupfile),
            goma_temp_dir.join("setup"),
        )
        with api.context(cwd=goma_temp_dir):
            api.docker.build(
                dockerfile=dockerfile,
                tags=[container_tag, container_tag + ":" + timestamp],
                build_args=["TOOLCHAIN=%s" % toolchain],
            )
            api.docker("push", container_tag)
            api.docker("push", container_tag + ":" + timestamp)
            push_toolchain_to_goma(
                api,
                goma_temp_dir,
                project,
                root,
                toolchain_tag,
                container_tag,
                stamp=timestamp,
            )


def RunSteps(
    api,
    package,
    version,
    toolchain,
    platforms,
    domain,
    project,
    root,
    update_goma_base_images,
):
    if not version:
        gitiles_commit = api.buildbucket.build_input.gitiles_commit
        if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id:
            version = str("git_revision:" + gitiles_commit.id)
    preprocess_platforms = platforms[:]
    if (
        "mac-amd64" in preprocess_platforms
        and "linux-amd64" not in preprocess_platforms
    ):
        # To cross compile mac target on goma linux workers, both mac and linux
        # toolchains are required.
        preprocess_platforms.append("linux-amd64")
    pkgs = api.cipd.EnsureFile()
    for platform in preprocess_platforms:
        pkgs.add_package(api.url.join(package, platform), version, platform)
    toolchain_dir = api.path["cleanup"].join("toolchain")
    with api.step.nest("ensure toolchain"):
        api.cipd.ensure(toolchain_dir, pkgs)

    timestamp = generate_time_stamp(api)
    if version.startswith("git_revision:"):
        version = version[len("git_revision:") :]

    goma_temp_dir = api.path["cleanup"].join("goma")
    ensure_gomatools(api, goma_temp_dir)
    with api.step.nest("preprocess toolchain"):
        if toolchain in ("clang", "gcc"):
            dest_dir = api.path["cleanup"].join("toolchain_copy")
            api.python(
                "preprocess %s" % toolchain,
                script=api.resource("preprocess_%s.py" % toolchain),
                args=[toolchain_dir, dest_dir,]
                + ["--platform=%s" % platform for platform in preprocess_platforms],
            )
            toolchain_dir = dest_dir
        for platform in preprocess_platforms:
            api.archive.package(toolchain_dir.join(platform)).archive(
                "pack toolchain package for %s" % platform,
                goma_temp_dir.join("{}-{}.tgz".format(toolchain, platform)),
            )

    ensure_goma_base_images(
        api, domain, project, root, timestamp, update_goma_base_images
    )
    api.docker(
        "pull", api.url.join(domain, project, root, "base"), step_name="pull base"
    )
    api.docker(
        "tag", api.url.join(domain, project, root, "base"), "base", step_name="tag base"
    )
    try:
        for platform in platforms:
            with api.step.nest("build toolchain image for %s" % platform):
                build_and_push_goma_toolchain(
                    api,
                    toolchain,
                    version,
                    domain,
                    project,
                    root,
                    platform,
                    goma_temp_dir,
                    timestamp,
                )
    finally:
        api.docker.cleanup()


def GenTests(api):
    for toolchain, version in [
        ("clang", "git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e"),
        (
            "gcc",
            "git_revision:3e7b85061947bdc7c7465743ba90734566860821,b5624945ea67525c0ba4ffec7a9d3f9366bf9071,5",
        ),
    ]:
        default_properties = api.properties(
            package="fuchsia/third_party/%s" % toolchain,
            version=version,
            toolchain=toolchain,
            domain="gcr.io",
            project="goma-fuchsia",
            root="fuchsia_linux",
        )

        properties_missing_platform = api.properties(
            package="fuchsia/third_party/%s/linux-amd64" % toolchain,
            version=version,
            toolchain=toolchain,
            domain="gcr.io",
            project="goma-fuchsia",
            root="fuchsia_linux",
        )

        wrong_platform_properties = api.properties(
            package="fuchsia/third_party/%s" % toolchain,
            version=version,
            toolchain=toolchain,
            platforms=["linux_amd64"],
            domain="gcr.io",
            project="goma-fuchsia",
            root="fuchsia_linux",
        )

        cross_compile_properties = api.properties(
            package="fuchsia/third_party/%s" % toolchain,
            version=version,
            toolchain=toolchain,
            platforms=["mac-amd64"],
            domain="gcr.io",
            project="goma-fuchsia",
            root="fuchsia_linux",
        )

        yield api.status_check.test(
            "%s_default" % toolchain
        ) + default_properties + api.buildbucket.try_build(
            git_repo="https://fuchsia.googlesource.com/integration"
        )

        yield api.status_check.test(
            "%s_base_image_missing" % toolchain
        ) + default_properties + api.buildbucket.try_build(
            git_repo="https://fuchsia.googlesource.com/integration"
        ) + api.step_data(
            "ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/runtime",
            retcode=1,
        ) + api.step_data(
            "ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/base",
            retcode=1,
        ) + api.step_data(
            "ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform",
            retcode=1,
        )

        yield api.status_check.test(
            "%s_wrong_platform" % toolchain
        ) + wrong_platform_properties + api.buildbucket.try_build(
            git_repo="https://fuchsia.googlesource.com/integration"
        )

        yield api.status_check.test(
            "%s_cross_compile" % toolchain
        ) + cross_compile_properties + api.buildbucket.try_build(
            git_repo="https://fuchsia.googlesource.com/integration"
        )

        default_properties_gitiles = api.properties(
            package="fuchsia/third_party/%s" % toolchain,
            toolchain=toolchain,
            domain="gcr.io",
            project="goma-fuchsia",
            root="fuchsia_linux",
        )
        yield (
            api.status_check.test("%s_default_with_buildset" % toolchain)
            + api.buildbucket.ci_build(
                bucket="ci",
                git_repo="https://fuchsia.googlesource.com/example",
                revision="a" * 40,
            )
            + default_properties_gitiles
        )