recipes/tools.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building and publishing tools."""

from recipe_engine.recipe_api import Property

PYTHON_VERSION_COMPATIBILITY = "PY3"

DEPS = [
    "fuchsia/cas_util",
    "fuchsia/checkout",
    "fuchsia/cipd_util",
    "fuchsia/go",
    "fuchsia/jiri",
    "fuchsia/status_check",
    "recipe_engine/buildbucket",
    "recipe_engine/context",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = {
    "project": Property(kind=str, help="Jiri remote manifest project", default=None),
    "manifest": Property(kind=str, help="Jiri manifest to use"),
    "remote": Property(kind=str, help="Remote manifest repository"),
    "output_name_map": Property(
        kind=dict, help="Mapping from tool to binary name", default={}
    ),
    "extra_arch_map": Property(
        kind=dict,
        help="Mapping from tool to list of extra platforms to build for",
        default={},
    ),
    "publish": Property(kind=bool, help="Whether to publish the tools", default=False),
}

# The current list of platforms that we build for.
GO_OS_ARCH = [
    ("linux", "amd64"),
    ("linux", "arm64"),
    ("darwin", "amd64"),
]


def list_main_packages(api, path):
    """Returns the 'main' go packages seq(str) on a given path (str)."""
    # Template to pass to `go list` under -f; the listed output will be in that
    # format. See https://golang.org/pkg/cmd/go/internal/list/ for more details.
    # While it might seem cleaner to use the -json list option over templating,
    # the output format is not parsable by python's json.loads()
    list_entry_template = "{{ .ImportPath  }},{{ .Name }}"
    list_entries = api.go(
        "list", "-f", list_entry_template, path, stdout=api.raw_io.output_text()
    ).stdout.splitlines()
    main_packages = []
    for entry in list_entries:
        pkg, name = entry.split(",")
        if name == "main":
            main_packages.append(pkg)
    return tuple(main_packages)


def tool_name(pkg_path):
    """Parses a package's source path to determine the associated tool's name."""
    tokens = pkg_path.split("/")
    assert "cmd" in tokens, "See https://github.com/golang-standards/project-layout#cmd"
    tokens.remove("cmd")
    # This assumes the main package is defined within a subdirectory structure of
    # $name/cmd/ or cmd/$name.
    return tokens[-1]


def RunSteps(api, project, manifest, remote, output_name_map, extra_arch_map, publish):
    with api.context(infra_steps=True):
        api.checkout.with_options(
            path=api.path["start_dir"],
            manifest=manifest,
            remote=remote,
            project=project,
        )
        project_info = api.checkout.project(project)

    revision = project_info["revision"]
    project_dir = api.path.abs_to_path(project_info["path"])
    staging_dir = api.path.mkdtemp("staging")

    with api.context(cwd=project_dir):
        # Build everything before running tests. Otherwise `go test` will itself
        # do the building and a build error might confusingly be presented as a
        # test error.
        api.go("build", "-v", "./...")
        api.go("test", "-v", "-race", "./...")

        for pkg in list_main_packages(api, "./..."):
            with api.step.nest(pkg):
                name = tool_name(pkg)
                for goos, goarch in GO_OS_ARCH + extra_arch_map.get(name, []):
                    env = {"GOOS": goos, "GOARCH": goarch}
                    platform = "%s-%s" % (goos.replace("darwin", "mac"), goarch)
                    with api.context(env=env), api.step.nest(platform):
                        # Rename the tool binary if a renaming is specified.
                        binary_name = output_name_map.get(name, name)
                        # Specially handle Windows because using the -o flag
                        # overrides the default behavior of adding the .exe
                        # extension.
                        if goos == "windows":
                            binary_name += ".exe"
                        binary_path = staging_dir.join(binary_name)
                        args = [
                            "build",
                            "-trimpath",
                            "-o",
                            binary_path,
                            "-gcflags=-trimpath=%s" % project_dir,
                            "-ldflags=-s -w -buildid=",
                            pkg,
                        ]
                        # Build the package.
                        api.go(*args)

                        # Upload the outputs.
                        if publish:
                            gitiles_commit = api.buildbucket.build.input.gitiles_commit
                            assert (
                                gitiles_commit.host
                            ), "we should only be uploading to CIPD in CI"

                            cipd_dir = gitiles_commit.host.split(".")[0].replace(
                                "-", "_"
                            )
                            api.cipd_util.upload_package(
                                "%s/%s/%s/%s"
                                % (
                                    cipd_dir,
                                    api.path.basename(project),
                                    name,
                                    platform,
                                ),
                                staging_dir,
                                [binary_path],
                                {"git_revision": revision},
                                repository=remote,
                            )
                        else:
                            api.cas_util.upload(staging_dir, output_property="isolated")


def GenTests(api):
    packages = [
        "go.fuchsia.dev/tools/cmd/gndoc",
        "go.fuchsia.dev/tools/symbolizer/cmd",
    ]
    output_name_map = {"gndoc": "renamed_gndoc"}
    extra_arch_map = {"gndoc": [("windows", "amd64")]}
    list_entries = ["%s,main" % pkg for pkg in packages]
    list_step_data = api.step_data(
        "go list", stdout=api.raw_io.output_text("\n".join(list_entries))
    )

    yield (
        api.status_check.test("publish")
        + api.buildbucket.ci_build(
            git_repo="https://fuchsia.googlesource.com/tools",
            revision=api.jiri.example_revision,
        )
        + api.properties(
            project="tools",
            manifest="tools",
            remote="https://fuchsia.googlesource.com/tools",
            output_name_map=output_name_map,
            extra_arch_map=extra_arch_map,
            publish=True,
        )
        + list_step_data
    )

    yield (
        api.status_check.test("ci")
        + api.buildbucket.ci_build(
            git_repo="https://fuchsia.googlesource.com/tools",
            revision=api.jiri.example_revision,
        )
        + api.properties(
            project="tools",
            manifest="tools",
            remote="https://fuchsia.googlesource.com/tools",
            output_name_map=output_name_map,
        )
        + list_step_data
    )

    yield (
        api.status_check.test("cq_try")
        + api.buildbucket.try_build(git_repo="https://fuchsia.googlesource.com/tools")
        + api.properties(
            project="tools",
            manifest="tools",
            remote="https://fuchsia.googlesource.com/tools",
            output_name_map=output_name_map,
        )
        + list_step_data
    )