recipes/contrib/gomatools_builder.py

# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building gomatools binaries."""

from recipe_engine.recipe_api import Property

DEPS = [
    "fuchsia/buildbucket_util",
    "fuchsia/cipd_util",
    "fuchsia/docker",
    "fuchsia/git_checkout",
    "fuchsia/go",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
    "recipe_engine/url",
]

PROPERTIES = {
    "package_prefix": Property(
        kind=str,
        help="cipd path prefix of goma_tools packages",
        default="fuchsia_internal/third_party/goma/server",
    ),
    "repository": Property(
        kind=str,
        help="git repository of goma server",
        default="https://chromium.googlesource.com/infra/goma/server",
    ),
    "source_root": Property(
        kind=str,
        help="directory name for goma server code",
        default="go.chromium.org/goma/server",
    ),
    "revision": Property(
        kind=str, help="git revision/branch of goma server repository", default="main"
    ),
    "domain": Property(
        kind=str, help="domain name of container registry", default="gcr.io"
    ),
    "project": Property(
        kind=str,
        help="target project of goma docker images",
        default="fuchsia-toolchain-images-gcr",
    ),
    "dry_run": Property(
        kind=bool, help="Whether to upload gomatools to CIPD.", default=True
    ),
}

PROTOC_VERSION = "protobuf_version:v3.8.0"


def RunSteps(
    api, package_prefix, source_root, repository, revision, domain, project, dry_run
):
    # Ensure protoc for pb file generation.
    pkgs = api.cipd.EnsureFile()
    pkgs.add_package("infra/tools/protoc/linux-amd64", PROTOC_VERSION)
    api.cipd.ensure(api.path["start_dir"].join("cipd"), pkgs)
    # Checkout Goma source
    go_path_dir = api.path["start_dir"].join("go")
    goma_server_dir = go_path_dir.join("src").join(*(source_root.split("/")))
    _, git_revision = api.git_checkout(
        repository, path=goma_server_dir, fallback_ref=revision
    )
    # Build goma server
    # TODO: Abstract building go tool logic into a fuchsia recipe module.
    with api.step.nest("build gomatools"):
        for item in api.file.listdir(
            name="list gomatools binaries",
            source=goma_server_dir.join("cmd"),
            test_data=["auth_server", "setup_cmd"],
        ):
            binary_name = api.path.basename(item)
            with api.context(
                cwd=goma_server_dir,
                env={
                    "GO111MODULE": "on",
                    "GOOS": "linux",
                    "GOARCH": "amd64",
                    "GOPATH": str(go_path_dir),
                    "GOCACHE": str(go_path_dir.join(".cache", "go-build")),
                },
            ):
                api.go(
                    "install",
                    api.path.join(source_root, "cmd", binary_name),
                    name="build " + binary_name,
                )
        with api.context(
            cwd=goma_server_dir,
            env={
                "GO111MODULE": "on",
                "GOOS": "windows",
                "GOARCH": "amd64",
                "GOPATH": str(go_path_dir),
                "GOCACHE": str(go_path_dir.join(".cache", "go-build")),
            },
        ):
            api.go(
                "build",
                "-o",
                go_path_dir.join("bin", "run.exe"),
                "remoteexec/winsupport/run.go",
                name="build windows wrapper",
            )
        # Generate service_descriptor.pb
        with api.context(
            cwd=goma_server_dir.join("proto"), env={"GOPATH": str(go_path_dir)}
        ):
            api.step(
                "generate service_descriptor.pb",
                [
                    api.path["start_dir"].join("cipd", "protoc"),
                    "-I.",
                    "--include_imports",
                    "--include_source_info",
                    "--descriptor_set_out",
                    go_path_dir.join("bin", "service_descriptor.pb"),
                    "exec/exec_service.proto",
                    "file/file_service.proto",
                    "execlog/log_service.proto",
                ],
            )
    # Upload gomatools to CIPD
    pkg_dir = go_path_dir.join("bin")
    if not dry_run:
        api.cipd_util.upload_package(
            package_prefix + "/gomatools/linux-amd64",
            pkg_dir,
            search_tag={"git_revision": git_revision},
            repository=repository,
        )
    # Build and push docker images
    image_stamp = "git_revision_" + git_revision
    with api.step.nest("copy docker files"):
        api.file.copy(
            "copy Dockerfile server-base",
            api.resource("Dockerfile.server-base"),
            go_path_dir.join("Dockerfile.server-base"),
        )
        api.file.copy(
            "copy Dockerfile server",
            api.resource("Dockerfile.server"),
            go_path_dir.join("Dockerfile.server"),
        )
        api.file.copy(
            "copy Dockerfile frontend",
            api.resource("Dockerfile.frontend"),
            go_path_dir.join("Dockerfile.frontend"),
        )
        api.file.copy(
            "copy Dockerfile gomatools",
            api.resource("Dockerfile.gomatools"),
            go_path_dir.join("Dockerfile.gomatools"),
        )
    with api.step.nest("build goma cluster images"):
        with api.context(cwd=go_path_dir):
            api.docker.build(
                dockerfile=go_path_dir.join("Dockerfile.server-base"),
                tags=["goma-server-base"],
            )
            for cmd in [
                "auth-server",
                "cache-server",
                "cmd-server",
                "execlog-server",
                "exec-server",
                "file-server",
            ]:
                build_server_image(
                    api, domain, project, cmd, "Dockerfile.server", image_stamp, dry_run
                )
            for cmd in [
                "frontend",
                "gomatools",
            ]:
                build_server_image(
                    api,
                    domain,
                    project,
                    cmd,
                    "Dockerfile.%s" % cmd,
                    image_stamp,
                    dry_run,
                )


def build_server_image(api, domain, project, cmd, dockerfile, image_stamp, dry_run):
    server_url = api.url.join(domain, project, cmd)
    api.docker.build(
        dockerfile=dockerfile,
        tags=[server_url, server_url + ":" + image_stamp],
        build_args=["app=%s" % cmd, "cmd=%s" % cmd.replace("-", "_")],
    )
    if not dry_run:
        api.docker("push", server_url, step_name="push %s" % server_url)
        api.docker(
            "push",
            server_url + ":" + image_stamp,
            step_name="push " + server_url + ":" + image_stamp,
        )


def GenTests(api):
    def properties(**kwargs):
        props = {
            "package_prefix": "fuchsia_internal/third_party/goma/server",
            "repository": "https://chromium.googlesource.com/infra/goma/server",
            "source_root": "go.chromium.org/goma/server",
            "dry_run": True,
        }
        props.update(**kwargs)
        return api.properties(**props)

    yield api.buildbucket_util.test("default") + properties()

    yield api.buildbucket_util.test("dry_run_off") + properties(dry_run=False)