recipes/tools.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building and publishing tools."""

from PB.recipes.fuchsia.tools import InputProperties

DEPS = [
    "fuchsia/buildbucket_util",
    "fuchsia/cas_util",
    "fuchsia/checkout",
    "fuchsia/cipd_util",
    "fuchsia/go",
    "recipe_engine/context",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = InputProperties

# The current list of platforms that we build for.
GO_OS_ARCH = [
    ("linux", "amd64"),
    ("linux", "arm64"),
    ("darwin", "amd64"),
    ("darwin", "arm64"),
]


def list_main_packages(api, path):
    """Returns the 'main' go packages seq(str) on a given path (str)."""
    # Template to pass to `go list` under -f; the listed output will be in that
    # format. See https://golang.org/pkg/cmd/go/internal/list/ for more details.
    # While it might seem cleaner to use the -json list option over templating,
    # the output format is not parsable by python's json.loads()
    list_entry_template = "{{ .ImportPath  }},{{ .Name }}"
    list_entries = api.go(
        "list",
        "-f",
        list_entry_template,
        path,
        stdout=api.raw_io.output_text(),
        step_test_data=lambda: api.raw_io.test_api.stream_output_text(
            "go.fuchsia.dev/tools/cmd/gndoc,main\n"
            + "go.fuchsia.dev/tools/symbolizer/cmd,main\n"
        ),
    ).stdout.splitlines()
    main_packages = []
    for entry in list_entries:
        pkg, name = entry.split(",")
        if name == "main":
            main_packages.append(pkg)
    return tuple(main_packages)


def tool_name(pkg_path):
    """Parses a package's source path to determine the associated tool's name."""
    tokens = pkg_path.split("/")
    assert "cmd" in tokens, "See https://github.com/golang-standards/project-layout#cmd"
    tokens.remove("cmd")
    # This assumes the main package is defined within a subdirectory structure of
    # $name/cmd/ or cmd/$name.
    return tokens[-1]


def RunSteps(api, props):
    checkout_dir = api.checkout.with_options(
        manifest=props.manifest,
        remote=props.remote,
        project=props.project,
    )
    project_info = api.checkout.project(props.project, checkout_root=checkout_dir)

    revision = project_info["revision"]
    project_dir = api.path.abs_to_path(project_info["path"])
    staging_dir = api.path.mkdtemp("staging")

    with api.context(cwd=project_dir):
        # Build everything before running tests. Otherwise `go test` will itself
        # do the building and a build error might confusingly be presented as a
        # test error.
        api.go("build", "-v", "./...")
        api.go("test", "-v", "-race", "./...")

        for pkg in list_main_packages(api, "./..."):
            with api.step.nest(pkg):
                name = tool_name(pkg)
                for goos, goarch in GO_OS_ARCH + list(
                    props.extra_arch_map.get(name, [])
                ):
                    env = {"GOOS": goos, "GOARCH": goarch}
                    platform = "%s-%s" % (goos.replace("darwin", "mac"), goarch)
                    with api.context(env=env), api.step.nest(platform):
                        # Rename the tool binary if a renaming is specified.
                        binary_name = props.output_name_map.get(name, name)
                        # Specially handle Windows because using the -o flag
                        # overrides the default behavior of adding the .exe
                        # extension.
                        if goos == "windows":
                            binary_name += ".exe"
                        binary_path = staging_dir.join(binary_name)
                        args = [
                            "build",
                            "-buildvcs=false",
                            "-trimpath",
                            "-o",
                            binary_path,
                            "-gcflags=-trimpath=%s" % project_dir,
                            "-ldflags=-s -w -buildid=",
                            pkg,
                        ]
                        # Build the package.
                        api.go(*args)

                        # Upload the outputs.
                        if props.publish:
                            assert props.cipd_root
                            api.cipd_util.upload_package(
                                "%s/%s/%s/%s"
                                % (
                                    props.cipd_root,
                                    api.path.basename(props.project),
                                    name,
                                    platform,
                                ),
                                staging_dir,
                                pkg_paths=[binary_path],
                                search_tag={"git_revision": revision},
                                repository=props.remote,
                            )
                        else:
                            api.cas_util.upload(staging_dir, output_property="isolated")


def GenTests(api):
    def properties(**kwargs):
        props = {
            "project": "tools",
            "manifest": "tools",
            "remote": "https://fuchsia.googlesource.com/tools",
            "output_name_map": {"gndoc": "renamed_gndoc"},
        }
        props.update(kwargs)
        return api.properties(**props)

    yield api.buildbucket_util.test("default") + properties()

    yield api.buildbucket_util.test("try", tryjob=True) + properties()

    yield (
        api.buildbucket_util.test("publish")
        + properties(
            extra_arch_map={"gndoc": [["windows", "amd64"]]},
            publish=True,
            cipd_root="fuchsia",
        )
    )