recipes/contrib/goma_client.py

# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building goma client binaries."""

from recipe_engine.recipe_api import Property

DEPS = [
    "fuchsia/buildbucket_util",
    "fuchsia/cipd_util",
    "fuchsia/git",
    "fuchsia/git_checkout",
    "fuchsia/macos_sdk",
    "fuchsia/python3",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = {
    "repository": Property(
        kind=str,
        help="git repository of goma client",
        default="https://chromium.googlesource.com/infra/goma/client",
    ),
    "revision": Property(
        kind=str, help="git revision/branch of goma server repository", default=""
    ),
    "skip_test": Property(
        kind=bool, help="Whether to skip Goma unit tests", default=False
    ),
    "dry_run": Property(
        kind=bool, help="Whether to upload goma client to CIPD.", default=False
    ),
}

GN_CIPD_PACKAGE = "gn/gn/${platform}"
GN_CIPD_VERSION = "git_revision:2e56c317bd8e2bf152cfa2ead6ac5fa476fe28b4"
NINJA_CIPD_PACKAGE = "infra/ninja/${platform}"
NINJA_CIPD_VERSION = "version:1.9.0"

GOMA_CLIENT_CIPD_PATH = "fuchsia/third_party/goma/client/${platform}"
DEPOT_TOOLS_GIT = "https://chromium.googlesource.com/chromium/tools/depot_tools.git"
DEFAULT_FUCHSIA_GOMA_BACKEND = "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog"
DEFAULT_FUCHSIA_GOMA_PORT = "8089"
DEFAULT_FUCHSIA_GOMA_IPC = "goma_fuchsia.ipc"
DEFAULT_FUCHSIA_GOMA_SUFFIX = "fuchsia"
DEFAULT_PROXY_PORT = "19081"

SCRIPT_TEST_DATA = """
('COMPILER_PROXY_SOCKET_NAME', 'goma.ipc'),
('COMPILER_PROXY_PORT', '8088'),
ports.append(os.environ.get('GOMA_COMPILER_PROXY_PORT', '8088'))
flags.update({'GOMA_SERVER_HOST': 'clients5.google.com'})
"""


def RunSteps(api, repository, revision, skip_test, dry_run):
    if not revision:
        gitiles_commit = api.buildbucket.build_input.gitiles_commit
        if (
            "https://%s/%s" % (gitiles_commit.host, gitiles_commit.project)
            == repository
        ):
            revision = gitiles_commit.id
        else:
            revision = "main"

    # Fetch gclient.
    with api.step.nest("fetch gclient"), api.context(infra_steps=True):
        depot_tools_path, _ = api.git_checkout(DEPOT_TOOLS_GIT)
    # Fetch goma-client
    with api.step.nest("fetch goma-client"):
        goma_client_path = api.path["start_dir"].join("goma")
        goma_src_path = goma_client_path.join("client")
        api.file.ensure_directory("makedirs goma", goma_client_path)

        with api.context(
            infra_steps=True,
            cwd=goma_client_path,
            env_prefixes={"PATH": [depot_tools_path]},
        ):
            api.step(
                "gclient config",
                [
                    "gclient",
                    "config",
                    "--name=client",
                    "--unmanaged",
                    "-v",
                    repository,
                ],
            )
            api.step(
                "gclient sync",
                ["gclient", "sync"],
            )

            with api.context(cwd=goma_src_path):
                api.git("pin revision", "checkout", revision)
                revision = api.git.get_hash()
                api.step(
                    "gclient sync",
                    ["gclient", "sync", "-v", "--output-json", api.json.output()],
                )

    with api.macos_sdk():
        with api.step.nest("build goma-client"):
            gn_path = api.cipd.ensure_tool(GN_CIPD_PACKAGE, GN_CIPD_VERSION)
            ninja_path = api.cipd.ensure_tool(NINJA_CIPD_PACKAGE, NINJA_CIPD_VERSION)

            # Build goma
            build_target = "Release"
            build_dir = goma_src_path.join("out").join(build_target)

            gn_args = [
                "is_debug=false",
                'cpu_arch="x64"',
                "enable_revision_check=true",
                "use_link_time_optimization=true",
                'default_server_host="%s"' % DEFAULT_FUCHSIA_GOMA_BACKEND,
                "default_compiler_proxy_port=%s" % DEFAULT_FUCHSIA_GOMA_PORT,
                'default_compiler_proxy_socket_name="%s"' % DEFAULT_FUCHSIA_GOMA_IPC,
                'default_project_suffix="%s"' % DEFAULT_FUCHSIA_GOMA_SUFFIX,
            ]

            patch_scripts(api, goma_src_path, "goma_ctl.py")
            patch_scripts(api, goma_src_path, "goma_auth.py")

            with api.context(cwd=goma_src_path):
                api.step(
                    "gn gen",
                    [
                        gn_path,
                        "--root=%s" % goma_src_path,
                        "gen",
                        build_dir,
                        "--args=%s" % " ".join(gn_args),
                    ],
                )
                api.step("ninja", [ninja_path, "-C", build_dir])

        # Run basic tests
        if not skip_test:
            api.python3(
                "tests",
                [
                    goma_src_path.join("build", "run_unittest.py"),
                    "--build-dir",
                    goma_src_path.join("out"),
                    "--target",
                    build_target,
                    "--non-stop",
                ],
            )

        # Create archive.
        api.python3(
            "archive",
            [
                goma_src_path.join("build", "archive.py"),
                "--platform",
                api.platform.name,
                "--build_dir",
                goma_src_path.join("out"),
                "--target_dir",
                build_target,
                "--dist_dir",
                api.path["tmp_base"],
            ],
        )

    # Upload to CIPD
    if not dry_run:
        pkg_dir = build_dir.join("goma-%s" % api.platform.name)
        api.cipd_util.upload_package(
            GOMA_CLIENT_CIPD_PATH,
            pkg_dir,
            search_tag={"git_revision": revision},
            repository=repository,
        )


def patch_scripts(api, goma_src_path, py_script):
    # goma_ctl.py has ipc and port number hard coded.
    # rewrite it with Fuchsia spec values.
    input_py = goma_src_path.join("client", py_script)
    api.step(
        name="rewrite %s " % py_script,
        cmd=[
            "python",
            api.resource("patch_script.py"),
            "--file",
            input_py,
            "--backend",
            DEFAULT_FUCHSIA_GOMA_BACKEND,
            "--ipc_socket",
            DEFAULT_FUCHSIA_GOMA_IPC,
            "--port",
            DEFAULT_FUCHSIA_GOMA_PORT,
            "--proxy_port",
            DEFAULT_PROXY_PORT,
        ],
    )


def GenTests(api):
    yield api.buildbucket_util.test(
        "default", git_repo="https://chromium.googlesource.com/infra/goma/client"
    )
    yield api.buildbucket_util.test("dry_run") + api.properties(dry_run=True)
    yield api.buildbucket_util.test("with_buildset", repo="example", revision="a" * 40)