recipes/tools.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building and publishing first-party Fuchsia Go tools.

Note: This recipe builds first-party host tools from go.fuchsia.dev/tools and does
not build third-party binaries, so third-party supply-chain baking delays do not apply.
"""

from __future__ import annotations

import functools
import re
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Any

from recipe_engine import post_process, recipe_api, recipe_test_api

from PB.recipes.fuchsia.tools import InputProperties

from RECIPE_MODULES.fuchsia import (
    buildbucket_util as buildbucket_util_module,
    cipd_util as cipd_util_module,
    git_checkout as git_checkout_module,
    go as go_module,
    raw_result as raw_result_module,
)
from RECIPE_MODULES.fuchsia.raw_result.api import wrap_run_steps
from RECIPE_MODULES.recipe_engine import (
    context as context_module,
    path as path_module,
    properties as properties_module,
    raw_io as raw_io_module,
    step as step_module,
)


@dataclass
class DEPS(recipe_api.RecipeScriptApi):
    cipd_util: cipd_util_module.API
    context: context_module.API
    git_checkout: git_checkout_module.API
    go: go_module.API
    path: path_module.API
    raw_io: raw_io_module.API
    raw_result: raw_result_module.API
    step: step_module.API


@dataclass
class TEST_DEPS(recipe_test_api.RecipeTestApi):
    buildbucket_util: buildbucket_util_module.TEST_API
    properties: properties_module.TEST_API


PROPERTIES = InputProperties

# The current list of platforms that we build for.
GO_OS_ARCH = [
    ("linux", "amd64"),
    ("linux", "arm64"),
    ("darwin", "amd64"),
    ("darwin", "arm64"),
]


@wrap_run_steps
def RunSteps(api: DEPS, props: InputProperties) -> None:
    assert not any(token in props.remote for token in ("third_party", "3pp")), (
        f"recipes/tools.py is intended strictly for first-party tools, got remote {props.remote!r}"
    )
    checkout_dir, revision = api.git_checkout(repo=props.remote)
    project_dir = checkout_dir

    staging_dir = api.path.mkdtemp("staging")

    with api.context(cwd=project_dir):
        # Build everything before running tests. Otherwise `go test` will itself
        # do the building and a build error might confusingly be presented as a
        # test error.
        api.go.build(["-v", "./..."])

        with api.step.nest("test"):
            for pkg in api.go.list(
                ["./..."],
                stdout=api.raw_io.output_text(),
                step_test_data=lambda: api.raw_io.test_api.stream_output_text(
                    "go.fuchsia.dev/tools/cmd/gndoc\n"
                    "go.fuchsia.dev/tools/symbolizer/cmd\n"
                ),
            ).stdout.splitlines():
                step = api.go.test(
                    ["-v", "-race", "-cover", pkg],
                    step_name=pkg,
                    stdout=api.raw_io.output_text(name="stdout", add_output_log=True),
                    step_test_data=functools.partial(
                        api.raw_io.test_api.stream_output_text,
                        f"? {pkg} [no test files]",
                    ),
                )

                # The last line of the verbose output is the one containing the
                # coverage information.
                step.presentation.step_text = coverage_percentage(
                    step.stdout.splitlines()[-1]
                )

        if not props.publish:
            return

        # Publish the main packages to cipd
        for pkg in list_main_packages(api, "./..."):
            with api.step.nest(pkg):
                name = tool_name(pkg)
                for goos, goarch in GO_OS_ARCH + list(
                    props.extra_arch_map.get(name, [])
                ):
                    env = {"GOOS": goos, "GOARCH": goarch}
                    platform = f"{goos.replace('darwin', 'mac')}-{goarch}"
                    with api.context(env=env), api.step.nest(platform):
                        # Rename the tool binary if a renaming is specified.
                        binary_name = props.output_name_map.get(name, name)
                        # Specially handle Windows because using the -o flag
                        # overrides the default behavior of adding the .exe
                        # extension.
                        if goos == "windows":
                            binary_name += ".exe"
                        binary_path = staging_dir / binary_name
                        api.go.build(
                            [
                                "-buildvcs=false",
                                "-trimpath",
                                "-o",
                                binary_path,
                                f"-gcflags=-trimpath={project_dir}",
                                "-ldflags=-s -w -buildid=",
                                pkg,
                            ],
                        )
                        # Upload the outputs.
                        assert props.cipd_root
                        api.cipd_util.upload_package(
                            "/".join([props.cipd_root, name, platform]),
                            staging_dir,
                            pkg_paths=[binary_path],
                            search_tag={"git_revision": revision},
                            repository=props.remote,
                        )


def list_main_packages(api: DEPS, path: str) -> tuple[str, ...]:
    """Returns the 'main' go packages seq(str) on a given path (str)."""
    # Template to pass to `go list` under -f; the listed output will be in that
    # format. See https://golang.org/pkg/cmd/go/internal/list/ for more details.
    # While it might seem cleaner to use the -json list option over templating,
    # the output format is not parsable by python's json.loads()
    list_entry_template = "{{ .ImportPath  }},{{ .Name }}"
    list_entries = api.go.list(
        [
            "-f",
            list_entry_template,
            path,
        ],
        stdout=api.raw_io.output_text(),
        step_test_data=lambda: api.raw_io.test_api.stream_output_text(
            "go.fuchsia.dev/tools/cmd/gndoc,main\n"
            "go.fuchsia.dev/tools/symbolizer/cmd,main\n"
        ),
    ).stdout.splitlines()
    main_packages = []
    for entry in list_entries:
        pkg, name = entry.split(",")
        if name == "main":
            main_packages.append(pkg)
    return tuple(main_packages)


def tool_name(pkg_path: str) -> str:
    """Parses a package's URL to determine the associated tool's name."""
    tokens = pkg_path.split("/")
    if "cmd" in tokens:
        # A package ending in "$name/cmd" should map to a tool named "$name".
        tokens.remove("cmd")
    return tokens[-1]


def coverage_percentage(coverage_line: str) -> str:
    # Regex to match on strings like:
    # 'ok  	go.fuchsia.dev/tools/gndoc	0.002s	coverage: 61.3% of statements'
    coverage_regex = re.compile(
        r"""^ok\s+ # match the initial ok
            \S+\s+ # any sequence of not spaces defines the pkg
            \S+\s+ # execution time
            (.*)$ # capture the last part which must be the coverage
            """,
        re.X,
    )

    match = coverage_regex.match(coverage_line)

    # [no test files] is the same string used by the go tool when a package has
    # no test files
    return match[1] if match else "[no test files]"


def GenTests(api: TEST_DEPS) -> Iterator[recipe_test_api.TestData]:
    def properties(**kwargs: Any) -> recipe_test_api.TestData:
        props = {
            "remote": "https://fuchsia.googlesource.com/tools",
            "output_name_map": {"gndoc": "renamed_gndoc"},
            "cipd_root": "fuchsia/tools",
        }
        props.update(kwargs)
        return api.properties(**props)

    yield api.buildbucket_util.test("default") + properties()

    yield api.buildbucket_util.test("try", tryjob=True) + properties()

    yield (
        api.buildbucket_util.test("publish")
        + properties(
            extra_arch_map={"gndoc": [["windows", "amd64"]]},
            publish=True,
        )
    )

    yield (
        api.buildbucket_util.test("third_party_remote_rejected")
        + properties(remote="https://fuchsia.googlesource.com/third_party/tools")
        + api.expect_exception("AssertionError")
        + api.post_process(
            post_process.SummaryMarkdownRE,
            r".*recipes/tools.py is intended strictly for first-party tools.*",
        )
        + api.post_process(post_process.DropExpectation)
    )

    yield (
        api.buildbucket_util.test("3pp_remote_rejected")
        + properties(remote="https://fuchsia.googlesource.com/infra/3pp")
        + api.expect_exception("AssertionError")
        + api.post_process(
            post_process.SummaryMarkdownRE,
            r".*recipes/tools.py is intended strictly for first-party tools.*",
        )
        + api.post_process(post_process.DropExpectation)
    )