recipes/tricium/tricium.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for running Tricium analyzers."""

import functools

from recipe_engine.post_process import MustRun
from PB.recipes.fuchsia.tricium.tricium import InputProperties

DEPS = [
    "fuchsia/build",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/git",
    "fuchsia/tricium_analyze",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/platform",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = InputProperties

# We'll skip running checks on any files for which this git attribute is
# explicitly unset. We *will* run checks on all files for which this attribute
# is unspecified, or explicitly set.
TRICIUM_GIT_ATTR = "tricium"


def RunSteps(api, props):
    checkout = api.checkout.fuchsia_with_options(
        manifest=props.manifest,
        remote=props.remote,
        # Tricium should always be triggered on CLs that affect projects in
        # the checkout, so we shouldn't skip patching any CLs.
        skip_patch_projects=(),
        use_incremental_cache=props.incremental_build,
    )

    project_name = api.buildbucket.build.input.gerrit_changes[0].project

    # If specified, download CIPD packages.
    if props.cipd_packages:
        with api.step.nest("ensure_packages"):
            with api.context(infra_steps=True):
                cipd_dir = checkout.root_dir.join("cipd")
                pkgs = api.cipd.EnsureFile()
                for package in props.cipd_packages:
                    pkgs.add_package(
                        package.name, package.version, subdir=package.subdir
                    )
                api.cipd.ensure(cipd_dir, pkgs)
                platform = "%s-%s" % (
                    api.platform.name.replace("win", "windows"),
                    {
                        "intel": {
                            32: "386",
                            # Note that this is different from the CIPD norm (this is how
                            # //prebuilt/third_party is laid out).
                            64: "x64",
                        },
                        "arm": {32: "armv6", 64: "arm64"},
                    }[api.platform.arch][api.platform.bits],
                )
                # Only these tools are being downloaded from CIPD, so directly set the
                # paths if cipd_packages is defined.
                api.tricium_analyze.black = cipd_dir.join("black")
                api.tricium_analyze.go = cipd_dir.join("go", platform, "bin", "go")
                api.tricium_analyze.gofmt = cipd_dir.join(
                    "go", platform, "bin", "gofmt"
                )
                api.tricium_analyze.yapf = cipd_dir.join("yapf")

    project_dir = api.path.abs_to_path(checkout.project(project_name)["path"])

    api.tricium_analyze.check_commit_message()

    if props.fint_params_path:
        api.tricium_analyze.build_results = api.build.with_options(
            checkout=checkout,
            fint_params_path=props.fint_params_path,
            incremental=props.incremental_build,
        )

    with api.context(cwd=project_dir):
        paths = api.git.get_changed_files(
            "get changed files",
            deleted=False,
            # If we include submodules then the output will include the
            # directory paths of those submodules, but tricium_analyze operates
            # on files, not directories. If we ever want to analyze changes to
            # files within submodules, we can instead do a submodule-aware diff
            # to get the paths to changed files within the submodule.
            ignore_submodules=True,
        )
        with api.step.nest("filter tricium-disabled files"):
            paths = filter_by_git_attrs(api, paths)

        api.tricium_analyze.suggest_fx = api.path.exists(
            checkout.root_dir.join("scripts", "fx")
        )
        api.tricium_analyze.checkout = checkout
        api.tricium_analyze(
            paths,
            enabled_analyzers=props.analyses,
            enabled_luci_analyzers=props.luci_analyzers,
        )


def filter_by_git_attrs(api, paths):
    """Remove files that have Tricium disabled by a git attribute."""
    filtered_files = []
    # Process long lists of files in chunks to avoid exceeding command length
    # limits. Arbitrarily chosen chunk size that's likely to not exceed the
    # limit.
    chunk_size = 100
    for i in range(0, len(paths), chunk_size):
        chunk = paths[i : i + chunk_size]
        mock_output = "\0".join(
            ["%s\0%s\0unspecified" % (f, TRICIUM_GIT_ATTR) for f in chunk]
        )
        step = api.git(
            "check git attributes for files %d-%d" % (i, i + len(chunk)),
            "check-attr",
            # Separate records with nuls, to prevent ugly escaping of non-ASCII
            # filenames.
            "-z",
            TRICIUM_GIT_ATTR,
            "--",
            *chunk,
            stdout=api.raw_io.output_text(),
            # Use functools.partial instead of a lambda to avoid Pylint's
            # cell-var-from-loop warning.
            step_test_data=functools.partial(
                api.raw_io.test_api.stream_output_text, mock_output
            ),
        )
        raw_records = step.stdout.strip("\0").split("\0")
        attr_statuses = [raw_records[i : i + 3] for i in range(0, len(raw_records), 3)]
        step.presentation.logs["file attributes"] = api.json.dumps(
            attr_statuses, indent=2
        ).splitlines()
        for filename, _, status in attr_statuses:
            if status != "unset":
                filtered_files.append(filename)

    return filtered_files


def GenTests(api):
    DIFF = """diff --git a/{0} b/{0}
index e684c1e..a76a10e 100644
--- a/{0}
+++ b/{0}
@@ -1,2 +1,4 @@
+  foo
+  bar
"""

    def changed_files_data(files):
        return api.git.get_changed_files("get changed files", files)

    def change_diff_data(filename):
        return api.step_data(
            "analyze %s.get change diff" % filename,
            api.raw_io.stream_output_text(DIFF.format(filename)),
        )

    def formatted_diff_data(filename):
        return api.step_data(
            "analyze %s.get formatted diff" % filename,
            api.raw_io.stream_output_text(DIFF.format(filename)),
        )

    def properties(cipd_packages=(), **kwargs):
        defaults = dict(
            manifest="flower",
            remote="https://fuchsia.googlesource.com/integration",
        )
        # Using CIPD packages implies that we don't need to do a build, and
        # vice versa.
        if cipd_packages:
            defaults["cipd_packages"] = cipd_packages
        else:
            defaults.update(fint_params_path="fint_params/tricium.textproto")
        defaults.update(kwargs)
        return api.properties(**defaults)

    yield (
        api.buildbucket_util.test("default", tryjob=True)
        + properties(
            analyses=["ClangFormat", "GNFormat"],
            luci_analyzers=["Spellchecker"],
            incremental_build=True,
        )
        + changed_files_data(["BUILD.gn", "hello.go"])
        + change_diff_data("BUILD.gn")
        + formatted_diff_data("BUILD.gn")
    )

    yield (
        api.buildbucket_util.test("with_cipd_packages", tryjob=True)
        + properties(
            analyses=["GoFmt"],
            cipd_packages=[
                {
                    "name": "fuchsia/go/${platform}",
                    "version": "integration",
                    "subdir": "go",
                }
            ],
        )
        + changed_files_data(["BUILD.gn", "hello.go"])
        + change_diff_data("hello.go")
        + formatted_diff_data("hello.go")
    )

    yield (
        api.buildbucket_util.test("attribute_disable", tryjob=True)
        + properties(analyses=["ClangFormat", "GNFormat"])
        + changed_files_data(["BUILD.gn", "hello.go"])
        + api.step_data(
            "filter tricium-disabled files.check git attributes for files 0-2",
            api.raw_io.stream_output_text(
                "BUILD.gn\0tricium\0unset\0hello.go\0tricium\0unspecified\0"
            ),
        )
    )

    # Even if running one analysis fails, we should still run the other
    # analyses and write their results before failing the build.
    yield (
        api.buildbucket_util.test("one_analysis_fails", tryjob=True, status="failure")
        + properties(analyses=["GNFormat", "GoFmt"])
        + changed_files_data(["hello.go", "BUILD.gn"])
        + change_diff_data("BUILD.gn")
        + formatted_diff_data("BUILD.gn")
        + change_diff_data("hello.go")
        + api.step_data("analyze hello.go.gofmt.run", retcode=1)
        + api.post_process(MustRun, "write results")
    )