recipes/tools.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building and publishing tools."""

import re
from PB.recipes.fuchsia.tools import InputProperties

DEPS = [
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/cipd_util",
    "fuchsia/git_checkout",
    "fuchsia/go",
    "recipe_engine/context",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = InputProperties

# The current list of platforms that we build for.
GO_OS_ARCH = [
    ("linux", "amd64"),
    ("linux", "arm64"),
    ("darwin", "amd64"),
    ("darwin", "arm64"),
]


def RunSteps(api, props):
    if props.manifest:
        checkout_dir = api.checkout.with_options(
            manifest=props.manifest,
            remote=props.remote,
            project=props.project,
        )
        project_info = api.checkout.project(props.project, checkout_root=checkout_dir)
        revision = project_info["revision"]
        project_dir = api.path.abs_to_path(project_info["path"])
    else:
        # If no jiri manifest is specified, checkout with pure git.
        checkout_dir, revision = api.git_checkout(repo=props.remote)
        project_dir = checkout_dir

    staging_dir = api.path.mkdtemp("staging")

    with api.context(cwd=project_dir):
        # Build everything before running tests. Otherwise `go test` will itself
        # do the building and a build error might confusingly be presented as a
        # test error.
        api.go.build(["-v", "./..."])

        with api.step.nest("test"):
            for pkg in api.go.list(
                ["./..."],
                stdout=api.raw_io.output_text(),
                step_test_data=lambda: api.raw_io.test_api.stream_output_text(
                    "go.fuchsia.dev/tools/cmd/gndoc\n"
                    "go.fuchsia.dev/tools/symbolizer/cmd\n"
                ),
            ).stdout.splitlines():
                step = api.go.test(
                    ["-v", "-race", "-cover", pkg],
                    step_name=pkg,
                    stdout=api.raw_io.output_text(name="stdout", add_output_log=True),
                    step_test_data=lambda: api.raw_io.test_api.stream_output_text(
                        f"? {pkg} [no test files]"
                    ),
                )

                # The last line of the verbose output is the one containing the
                # coverage information.
                step.presentation.step_text = coverage_percentage(
                    step.stdout.splitlines()[-1]
                )

        if not props.publish:
            return

        # Publish the main packages to cipd
        for pkg in list_main_packages(api, "./..."):
            with api.step.nest(pkg):
                name = tool_name(pkg)
                for goos, goarch in GO_OS_ARCH + list(
                    props.extra_arch_map.get(name, [])
                ):
                    env = {"GOOS": goos, "GOARCH": goarch}
                    platform = f"{goos.replace('darwin', 'mac')}-{goarch}"
                    with api.context(env=env), api.step.nest(platform):
                        # Rename the tool binary if a renaming is specified.
                        binary_name = props.output_name_map.get(name, name)
                        # Specially handle Windows because using the -o flag
                        # overrides the default behavior of adding the .exe
                        # extension.
                        if goos == "windows":
                            binary_name += ".exe"
                        binary_path = staging_dir / binary_name
                        api.go.build(
                            [
                                "-buildvcs=false",
                                "-trimpath",
                                "-o",
                                binary_path,
                                f"-gcflags=-trimpath={project_dir}",
                                "-ldflags=-s -w -buildid=",
                                pkg,
                            ],
                        )
                        # Upload the outputs.
                        assert props.cipd_root
                        pkg_name_parts = [props.cipd_root]
                        if props.project:
                            pkg_name_parts.append(props.project.split("/")[-1])
                        pkg_name_parts.extend([name, platform])
                        api.cipd_util.upload_package(
                            "/".join(pkg_name_parts),
                            staging_dir,
                            pkg_paths=[binary_path],
                            search_tag={"git_revision": revision},
                            repository=props.remote,
                        )


def list_main_packages(api, path):
    """Returns the 'main' go packages seq(str) on a given path (str)."""
    # Template to pass to `go list` under -f; the listed output will be in that
    # format. See https://golang.org/pkg/cmd/go/internal/list/ for more details.
    # While it might seem cleaner to use the -json list option over templating,
    # the output format is not parsable by python's json.loads()
    list_entry_template = "{{ .ImportPath  }},{{ .Name }}"
    list_entries = api.go.list(
        [
            "-f",
            list_entry_template,
            path,
        ],
        stdout=api.raw_io.output_text(),
        step_test_data=lambda: api.raw_io.test_api.stream_output_text(
            "go.fuchsia.dev/tools/cmd/gndoc,main\n"
            "go.fuchsia.dev/tools/symbolizer/cmd,main\n"
        ),
    ).stdout.splitlines()
    main_packages = []
    for entry in list_entries:
        pkg, name = entry.split(",")
        if name == "main":
            main_packages.append(pkg)
    return tuple(main_packages)


def tool_name(pkg_path):
    """Parses a package's URL to determine the associated tool's name."""
    tokens = pkg_path.split("/")
    if "cmd" in tokens:
        # A package ending in "$name/cmd" should map to a tool named "$name".
        tokens.remove("cmd")
    return tokens[-1]


def coverage_percentage(coverage_line):
    # Regex to match on strings like:
    # 'ok  	go.fuchsia.dev/tools/gndoc	0.002s	coverage: 61.3% of statements'
    coverage_regex = re.compile(
        r"""^ok\s+ # match the initial ok
            \S+\s+ # any sequence of not spaces defines the pkg
            \S+\s+ # execution time
            (.*)$ # capture the last part which must be the coverage
            """,
        re.X,
    )

    match = coverage_regex.match(coverage_line)

    # [no test files] is the same string used by the go tool when a package has
    # no test files
    return match[1] if match else "[no test files]"


def GenTests(api):
    def properties(**kwargs):
        props = {
            "remote": "https://fuchsia.googlesource.com/tools",
            "output_name_map": {"gndoc": "renamed_gndoc"},
        }
        props.update(kwargs)
        return api.properties(**props)

    yield api.buildbucket_util.test("default") + properties()

    yield api.buildbucket_util.test("try", tryjob=True) + properties()

    yield (
        api.buildbucket_util.test("publish__jiri")
        + properties(
            extra_arch_map={"gndoc": [["windows", "amd64"]]},
            publish=True,
            cipd_root="fuchsia",
            manifest="tools",
            project="foo/tools",
        )
    )