recipes/goma_client.py

# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building goma client binaries."""

from recipe_engine.recipe_api import Property

DEPS = [
    "fuchsia/git",
    "fuchsia/gn",
    "fuchsia/macos_sdk",
    "fuchsia/ninja",
    "fuchsia/status_check",
    "fuchsia/upload",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/properties",
    "recipe_engine/python",
    "recipe_engine/step",
]

PROPERTIES = {
    "repository": Property(
        kind=str,
        help="git repository of goma client",
        default="https://chromium.googlesource.com/infra/goma/client",
    ),
    "revision": Property(
        kind=str, help="git revision/branch of goma server repository", default=""
    ),
    "dry_run": Property(
        kind=bool, help="Whether to upload goma client to CIPD.", default=False
    ),
}

GOMA_CLIENT_CIPD_PATH = "fuchsia/third_party/goma/client/${platform}"
DEPOT_TOOLS_GIT = "https://chromium.googlesource.com/chromium/tools/depot_tools.git"
GN_CIPD_PATH = "gn/gn/${platform}"
GN_CIPD_REVISION = "git_revision:222ae505623476f08fd5db3544d7a9e6a0b50472"
NINJA_CIPD_PATH = "infra/ninja/${platform}"
NINJA_CIPD_REVISION = "version:1.9.0"
DEFAULT_FUCHSIA_GOMA_BACKEND = "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog"
DEFAULT_FUCHSIA_GOMA_PORT = "8089"
DEFAULT_FUCHSIA_GOMA_IPC = "goma_fuchsia.ipc"
DEFAULT_FUCHSIA_GOMA_SUFFIX = "fuchsia"

SCRIPT_TEST_DATA = """
('COMPILER_PROXY_SOCKET_NAME', 'goma.ipc'),
('COMPILER_PROXY_PORT', '8088'),
ports.append(os.environ.get('GOMA_COMPILER_PROXY_PORT', '8088'))
flags.update({'GOMA_SERVER_HOST': 'clients5.google.com'})
"""


def patch_scripts(api, goma_src_path, py_script):
    # goma_ctl.py has ipc and port number hard coded.
    # rewrite it with Fuchsia spec values.
    input_py = goma_src_path.join("client", py_script)
    api.python(
        name="rewrite %s " % py_script,
        script=api.resource("patch_script.py"),
        args=[
            "--file",
            input_py,
            "--backend",
            DEFAULT_FUCHSIA_GOMA_BACKEND,
            "--ipc_socket",
            DEFAULT_FUCHSIA_GOMA_IPC,
            "--port",
            DEFAULT_FUCHSIA_GOMA_PORT,
        ],
    )


def RunSteps(api, repository, revision, dry_run):
    # Ensure GN
    pkgs = api.cipd.EnsureFile()
    pkgs.add_package(GN_CIPD_PATH, GN_CIPD_REVISION)
    pkgs.add_package(NINJA_CIPD_PATH, NINJA_CIPD_REVISION)
    cipd_dir = api.path["start_dir"].join("cipd")
    api.cipd.ensure(cipd_dir, pkgs)
    api.gn.set_path(cipd_dir.join("gn"))
    api.ninja.set_path(cipd_dir.join("ninja"))

    if not revision:
        gitiles_commit = api.buildbucket.build_input.gitiles_commit
        if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id:
            revision = gitiles_commit.id
        else:
            revision = "master"

    # Fetch gclient.
    with api.step.nest("fetch gclient"):
        depot_tools_path = api.path["start_dir"].join("depot_tools")
        with api.context(infra_steps=True):
            api.git.checkout(DEPOT_TOOLS_GIT, depot_tools_path)
    # Fetch goma-client
    with api.step.nest("fetch goma-client"):
        goma_client_path = api.path["start_dir"].join("goma")
        goma_src_path = goma_client_path.join("client")
        api.file.ensure_directory("makedirs goma", goma_client_path)

        with api.context(
            infra_steps=True,
            cwd=goma_client_path,
            env_prefixes={"PATH": [depot_tools_path]},
        ):
            api.step(
                "gclient config",
                [
                    "gclient",
                    "config",
                    "--name=client",
                    "--unmanaged",
                    "-v",
                    repository,
                ],
            )
            api.step(
                "gclient sync", ["gclient", "sync",],
            )

            with api.context(cwd=goma_src_path):
                api.git("pin revision", "checkout", revision)
                revision = api.git.get_hash()
                api.step(
                    "gclient sync",
                    ["gclient", "sync", "-v", "--output-json", api.json.output()],
                )

    with api.macos_sdk():
        with api.step.nest("build goma-client"):
            # Build goma
            build_target = "Release"
            build_dir = goma_src_path.join("out").join(build_target)

            gn_args = [
                "is_debug=false",
                'cpu_arch="x64"',
                "enable_revision_check=true",
                "use_link_time_optimization=true",
                'default_server_host="%s"' % DEFAULT_FUCHSIA_GOMA_BACKEND,
                "default_compiler_proxy_port=%s" % DEFAULT_FUCHSIA_GOMA_PORT,
                'default_compiler_proxy_socket_name="%s"' % DEFAULT_FUCHSIA_GOMA_IPC,
                'default_project_suffix="%s"' % DEFAULT_FUCHSIA_GOMA_SUFFIX,
            ]

            patch_scripts(api, goma_src_path, "goma_ctl.py")
            patch_scripts(api, goma_src_path, "goma_auth.py")

            with api.context(cwd=goma_src_path):
                api.gn(
                    "--root=%s" % goma_src_path,
                    "gen",
                    build_dir,
                    "--args=%s" % " ".join(gn_args),
                    name="gn gen",
                )
                api.ninja(
                    build_dir, [],
                )
        # Run basic tests
        api.python(
            name="tests",
            script=goma_src_path.join("build", "run_unittest.py"),
            args=[
                "--build-dir",
                goma_src_path.join("out"),
                "--target",
                build_target,
                "--non-stop",
            ],
        )

        # Create archive.
        api.python(
            name="archive",
            script=goma_src_path.join("build", "archive.py"),
            args=[
                "--platform",
                api.platform.name,
                "--build_dir",
                goma_src_path.join("out"),
                "--target_dir",
                build_target,
                "--dist_dir",
                api.path["tmp_base"],
            ],
        )

    # Upload to CIPD
    if not dry_run:
        pkg_dir = build_dir.join("goma-%s" % api.platform.name)
        pkg_file = api.path["tmp_base"].join("package.cipd")
        api.upload.cipd_package(
            GOMA_CLIENT_CIPD_PATH,
            pkg_dir,
            [api.upload.DirectoryPath(pkg_dir)],
            {"git_revision": revision},
            repository,
        )


def GenTests(api):
    yield api.status_check.test("default") + api.buildbucket.try_build(
        git_repo="https://fuchsia.googlesource.com/integration"
    )
    yield api.status_check.test("dry_run") + api.properties(
        dry_run=True
    ) + api.buildbucket.try_build(
        git_repo="https://fuchsia.googlesource.com/integration"
    )
    yield (
        api.status_check.test("default_with_buildset")
        + api.buildbucket.ci_build(
            bucket="ci",
            git_repo="https://fuchsia.googlesource.com/example",
            revision="a" * 40,
        )
    )