recipes/tricium/tricium.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for running Tricium analyzers."""

import functools

from recipe_engine.config import ConfigGroup, ConfigList, List, Single
from recipe_engine.post_process import MustRun
from recipe_engine.recipe_api import Property

DEPS = [
    "fuchsia/checkout",
    "fuchsia/git",
    "fuchsia/status_check",
    "fuchsia/tricium_analyze",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = {
    "manifest": Property(kind=str, help="Jiri manifest to use", default=None),
    "remote": Property(kind=str, help="Manifest project remote", default=None),
    "cipd_packages": Property(
        help="CIPD packages containing necessary binaries.",
        kind=ConfigList(
            lambda: ConfigGroup(
                name=Single(str),
                version=Single(str),
                # Language subdirectory (e.g. clang) in which to put the
                # package. The Tricium modules assume a Fuchsia prebuilts-like
                # layout.
                subdir=Single(str),
            )
        ),
        default=(),
    ),
    "board": Property(kind=str, help="Board to build", default=""),
    "build_type": Property(kind=str, help="The build type", default=""),
    "product": Property(kind=str, help="Product to build", default=""),
    "fint_params_path": Property(
        kind=str, help="Path to a fint params file", default=None
    ),
    "target": Property(kind=str, help="Target to build", default=""),
}

# We'll skip running checks on any files for which this git attribute is
# explicitly unset. We *will* run checks on all files for which this attribute
# is unspecified, or explicitly set.
TRICIUM_GIT_ATTR = "tricium"


def RunSteps(
    api,
    manifest,
    remote,
    cipd_packages,
    board,
    build_type,
    product,
    fint_params_path,
    target,
):
    with api.context(infra_steps=True):
        checkout_root = api.path["start_dir"]
        checkout = api.checkout.fuchsia_with_options(
            path=checkout_root,
            manifest=manifest,
            remote=remote,
            # Tricium should always be triggered on CLs that affect projects in
            # the checkout, so we shouldn't skip patching any CLs.
            skip_patch_projects=(),
        )

    project_name = api.buildbucket.build.input.gerrit_changes[0].project

    # If specified, download CIPD packages.
    if cipd_packages:
        with api.step.nest("ensure_packages"):
            with api.context(infra_steps=True):
                cipd_dir = checkout_root.join("cipd")
                pkgs = api.cipd.EnsureFile()
                for package in cipd_packages:
                    pkgs.add_package(
                        package["name"], package["version"], subdir=package["subdir"]
                    )
                api.cipd.ensure(cipd_dir, pkgs)
                platform = "%s-%s" % (
                    api.platform.name.replace("win", "windows"),
                    {
                        "intel": {
                            32: "386",
                            # Note that this is different from the CIPD norm (this is how
                            # //prebuilt/third_party is laid out).
                            64: "x64",
                        },
                        "arm": {32: "armv6", 64: "arm64",},
                    }[api.platform.arch][api.platform.bits],
                )
                # Only these tools are being downloaded from CIPD, so directly set the
                # paths if cipd_packages is defined.
                api.tricium_analyze.black = cipd_dir.join("black")
                api.tricium_analyze.go = cipd_dir.join("go", platform, "bin", "go")
                api.tricium_analyze.gofmt = cipd_dir.join(
                    "go", platform, "bin", "gofmt"
                )
                api.tricium_analyze.yapf = cipd_dir.join("yapf")

    project_dir = checkout_root.join(checkout.project(project_name)["path"])

    api.tricium_analyze.check_commit_message()

    with api.context(cwd=project_dir):
        paths = api.git.get_changed_files("get changed files", deleted=False)
        with api.step.nest("filter tricium-disabled files"):
            paths = filter_by_git_attrs(api, paths)

        api.tricium_analyze.suggest_fx = api.path.exists(
            checkout_root.join("scripts", "fx")
        )
        api.tricium_analyze.checkout = checkout
        api.tricium_analyze(
            paths,
            board=board,
            build_type=build_type,
            product=product,
            fint_params_path=fint_params_path,
            target=target,
        )


def filter_by_git_attrs(api, paths):
    """Remove files that have Tricium disabled by a git attribute."""
    filtered_files = []
    # Process long lists of files in chunks to avoid exceeding command length
    # limits. Arbitrarily chosen chunk size that's likely to not exceed the
    # limit.
    chunk_size = 100
    for i in range(0, len(paths), chunk_size):
        chunk = paths[i : i + chunk_size]
        mock_output = "\0".join(
            ["%s\0%s\0unspecified" % (f, TRICIUM_GIT_ATTR) for f in chunk]
        )
        step = api.git(
            "check git attributes for files %d-%d" % (i, i + len(chunk)),
            "check-attr",
            # Separate records with nuls, to prevent ugly escaping of non-ASCII
            # filenames.
            "-z",
            TRICIUM_GIT_ATTR,
            "--",
            *chunk,
            stdout=api.raw_io.output(),
            # Use functools.partial instead of a lambda to avoid Pylint's
            # cell-var-from-loop warning.
            step_test_data=functools.partial(
                api.raw_io.test_api.stream_output, mock_output
            )
        )
        raw_records = step.stdout.strip("\0").split("\0")
        attr_statuses = [raw_records[i : i + 3] for i in range(0, len(raw_records), 3)]
        step.presentation.logs["file attributes"] = api.json.dumps(
            attr_statuses, indent=2
        ).splitlines()
        for filename, _, status in attr_statuses:
            if status != "unset":
                filtered_files.append(filename)

    return filtered_files


def GenTests(api):
    DIFF = """diff --git a/{0} b/{0}
index e684c1e..a76a10e 100644
--- a/{0}
+++ b/{0}
@@ -1,2 +1,4 @@
+  foo
+  bar
"""

    def changed_files_data(files):
        return api.git.get_changed_files("get changed files", files)

    def change_diff_data(filename):
        return api.step_data(
            "analyze %s.get change diff" % filename,
            api.raw_io.stream_output(DIFF.format(filename)),
        )

    def formatted_diff_data(filename):
        return api.step_data(
            "analyze %s.get formatted diff" % filename,
            api.raw_io.stream_output(DIFF.format(filename)),
        )

    try_build = api.buildbucket.try_build(
        git_repo="https://fuchsia.googlesource.com/topaz"
    )

    def properties(cipd_packages=(), **kwargs):
        defaults = dict(
            manifest="flower",
            project="integration",
            remote="https://fuchsia.googlesource.com/integration",
            ref="refs/changes/12345/2",
        )
        # Using CIPD packages implies that we don't need to do a build, and
        # vice versa.
        if cipd_packages:
            defaults["cipd_packages"] = cipd_packages
        else:
            defaults.update(
                dict(
                    fint_params_path="specs/tricium.fint.textproto",
                    target="x64",
                    build_type="release",
                    board="boards/x64.gni",
                    product="products/core.gni",
                )
            )
        defaults.update(kwargs)
        return api.properties(**defaults)

    yield (
        api.status_check.test("default")
        + try_build
        + properties(analyses=["ClangFormat", "GNFormat"])
        + changed_files_data(["BUILD.gn", "hello.go"])
        + change_diff_data("BUILD.gn")
        + formatted_diff_data("BUILD.gn")
    )

    yield (
        api.status_check.test("with_cipd_packages")
        + try_build
        + properties(
            analyses=["GoFmt"],
            cipd_packages=[
                {
                    "name": "fuchsia/go/${platform}",
                    "version": "integration",
                    "subdir": "go",
                }
            ],
        )
        + changed_files_data(["BUILD.gn", "hello.go"])
        + change_diff_data("hello.go")
        + formatted_diff_data("hello.go")
    )

    yield (
        api.status_check.test("attribute_disable")
        + try_build
        + properties(analyses=["ClangFormat", "GNFormat"])
        + changed_files_data(["BUILD.gn", "hello.go"])
        + api.step_data(
            "filter tricium-disabled files.check git attributes for files 0-2",
            api.raw_io.stream_output(
                "BUILD.gn\0tricium\0unset\0hello.go\0tricium\0unspecified\0"
            ),
        )
    )

    # Even if running one analysis fails, we should still run the other
    # analyses and write their results before failing the build.
    yield (
        api.status_check.test("one_analysis_fails", status="failure")
        + try_build
        + properties(analyses=["GNFormat", "GoFmt"])
        + changed_files_data(["hello.go", "BUILD.gn"])
        + change_diff_data("BUILD.gn")
        + formatted_diff_data("BUILD.gn")
        + change_diff_data("hello.go")
        + api.step_data("analyze hello.go.gofmt.run", retcode=1)
        + api.post_process(MustRun, "write results")
    )