recipes/static_checks.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Recipe for checks that require a checkout but not a full build."""


from RECIPE_MODULES.fuchsia.utils import pluralize
from PB.recipes.fuchsia.static_checks import InputProperties

PYTHON_VERSION_COMPATIBILITY = "PY3"

DEPS = [
    "fuchsia/autocorrelator",
    "fuchsia/build",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/utils",
    "recipe_engine/context",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    checkout = api.checkout.fuchsia_with_options(
        manifest=props.manifest,
        remote=props.remote,
    )

    with api.autocorrelator.context(ci_base_commit=checkout.integration_revision):
        build_results = api.build.with_options(
            checkout=checkout, fint_params_path=props.fint_params_path
        )
        with api.context(cwd=checkout.root_dir):
            check_docs(api, build_results.tool("doc-checker"))
            check_gn_formatting(api, checkout, build_results.tool("gn"))
            check_licenses(
                api, build_results.tool("check-licenses"), props.check_licenses_configs
            )


def check_docs(api, doc_checker_tool):
    """Verify structure and content of documentation files.

    Args:
        doc_checker_tool (Path): Path to doc-checker tool.

    Raises:
        StepFailure: One or more errors found in documentation.
    """
    step = api.step(
        "doc check",
        [doc_checker_tool, "--local-links-only"],
        stdout=api.raw_io.output_text(),
        ok_ret="any",
    )
    step.presentation.logs["stdout"] = step.stdout
    if step.retcode:
        step.presentation.status = api.step.FAILURE
        raise api.step.StepFailure(
            api.buildbucket_util.summary_message(
                step.stdout,
                "(failure summary truncated, see `doc check` stdout for full "
                "failure details)",
            )
        )


def check_gn_formatting(api, checkout, gn_tool):
    """Check gn formatting.

    Does not attempt to suggest fixes; the tricium recipe does that.

    Args:
        checkout (CheckoutResults): CheckoutResults object.
        gn_tool (Path): Path to gn tool.

    Raises:
        StepFailure: One or more GN files are not formatted properly.
    """
    with api.step.nest("gn format"):
        # On try jobs, only check the modified files. On CI, check them all.
        if api.buildbucket_util.is_tryjob:
            files = checkout.changed_files(
                test_data=["bar/BUILD.gn", "foo.cc", "third_party/foo/BUILD.gn"],
                deleted=False,
            )
        else:
            files = checkout.list_files(
                file="*.gn*",
                test_data="bar/BUILD.gn\nthird_party/foo/BUILD.gn",
            )
            files = [str(api.path.relpath(f, checkout.root_dir)) for f in files]
        files = [
            f for f in files if f.endswith((".gn", ".gni")) and "third_party" not in f
        ]
        if files:
            format_step = api.step(
                "gn format --dry-run",
                [gn_tool, "format", "--dry-run"] + files,
                stdout=api.raw_io.output_text(),
                ok_ret=(0, 2),
            )
            format_step.presentation.logs["stdout"] = format_step.stdout
            # Retcode 2 indicates that formatting is successful but differs
            # from disk. See `gn help format` for more information.
            if format_step.retcode == 2:
                format_step.presentation.status = api.step.FAILURE
                unformatted_files = format_step.stdout.strip("\n").split("\n")
                header = "%s not formatted: \n" % pluralize(
                    "file", len(unformatted_files)
                )
                error_lines = [header] + [
                    "- %s" % api.path.relpath(f, checkout.root_dir)
                    for f in unformatted_files
                ]
                raise api.step.StepFailure(
                    api.buildbucket_util.summary_message(
                        "\n".join(error_lines),
                        "(failure summary truncated, see `gn format --dry-run` "
                        "stdout for full failure details)",
                        escape_markdown=False,
                    )
                )


def check_licenses(api, check_licenses_tool, check_licenses_configs):
    """Check the validity of licenses.

    Args:
        check_licenses_tool (Path): Path to gn tool.
        check_licenses_configs (seq(str)): See corresponding recipe property
          docstring.

    Raises:
        StepFailure: One or more errors found in licenses.
    """
    check_licenses_command = [
        check_licenses_tool,
        "-output_license_file=false",
    ]
    if check_licenses_configs:
        check_licenses_command.append(
            "-config_file=%s" % ",".join(check_licenses_configs)
        )
    step = api.step(
        "check licenses",
        check_licenses_command,
        stderr=api.raw_io.output_text(),
        ok_ret="any",
    )
    step.presentation.logs["stderr"] = step.stderr
    if step.retcode:
        step.presentation.status = api.step.FAILURE
        raise api.step.StepFailure(
            api.buildbucket_util.summary_message(
                step.stderr,
                "(failure summary truncated, see `check licenses` stderr for "
                "full failure details)",
            )
        )


def GenTests(api):
    source_info = [
        {
            "name": "integration",
            "remote": "https://fuchsia.googlesource.com/integration",
            "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738",
            "relativePath": "integration",
        },
        {
            "name": "fuchsia",
            "remote": "https://fuchsia.googlesource.com/fuchsia",
            "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738",
            "relativePath": ".",
        },
    ]

    def props(**kwargs):
        return api.properties(
            manifest="fuchsia",
            remote="https://fuchsia.googlesource.com/integration",
            fint_params_path="specs/static-checks.textproto",
            **kwargs
        )

    yield (
        api.buildbucket_util.test("default_ci", tryjob=False)
        + props(check_licenses_configs=["config1.json", "config2.json"])
        + api.checkout.source_info(source_info)
    )

    yield (
        api.buildbucket_util.test("default_cq", tryjob=True)
        + props()
        + api.checkout.source_info(source_info)
    )

    yield (
        api.buildbucket_util.test("failed_build", tryjob=True, status="failure")
        + props()
        + api.checkout.source_info(source_info)
        + api.step_data("build.ninja", retcode=1)
    )

    yield (
        api.buildbucket_util.test("failed_doc_check", tryjob=True, status="failure")
        + props()
        + api.checkout.source_info(source_info)
        + api.step_data(
            "doc check", api.raw_io.stream_output_text("Found 2 errors."), retcode=1
        )
    )

    yield (
        api.buildbucket_util.test("failed_gn_format", tryjob=True, status="failure")
        + props()
        + api.checkout.source_info(source_info)
        + api.step_data(
            "gn format.gn format --dry-run",
            api.raw_io.stream_output_text("foo/BUILD.gn\nbar/BUILD.gn\n"),
            retcode=2,
        )
    )

    yield (
        api.buildbucket_util.test(
            "failed_licenses_check", tryjob=True, status="failure"
        )
        + props()
        + api.checkout.source_info(source_info)
        + api.step_data(
            "check licenses",
            stderr=api.raw_io.output_text("Encountered prohibited license types."),
            retcode=1,
        )
    )