recipes/static_checks.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Recipe for checks that require a checkout but not a full build."""


from RECIPE_MODULES.fuchsia.utils import pluralize
from PB.recipes.fuchsia.static_checks import InputProperties
from PB.infra.metadata import Metadata

DEPS = [
    "fuchsia/autocorrelator",
    "fuchsia/build",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/git",
    "fuchsia/reported_step",
    "fuchsia/shac",
    "fuchsia/utils",
    "recipe_engine/buildbucket",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/proto",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    checkout = api.checkout.fuchsia_with_options(
        manifest=props.manifest,
        remote=props.remote,
        use_incremental_cache=props.incremental_build,
        enable_submodules=props.enable_submodules,
    )

    revision = checkout.integration_revision
    gerrit_project = None
    if changes := api.buildbucket.build.input.gerrit_changes:
        gerrit_project = changes[0].project

    with api.autocorrelator.context(ci_base_commit=revision):
        build_results = api.build.with_options(
            checkout=checkout,
            fint_params_path=props.fint_params_path,
            incremental=props.incremental_build,
        )
        with api.context(cwd=checkout.root_dir):
            check_licenses(api, build_results.tool("check-licenses"))
            check_metadata(api, checkout)
        project_dirs = ["."] + list(props.additional_shac_project_dirs)
        for project_dir in project_dirs:
            step_name = "run shac checks"
            if project_dir != ".":
                step_name += f" in {project_dir}"
            with api.step.nest(step_name), api.context(
                # Checks in sub-repos depend on the $FUCHSIA_DIR env var to
                # resolve the fuchsia checkout root so it can be passed through
                # the filesystem sandbox, which would normally prevent checks in
                # one repository from accessing files in parent directories.
                env={"FUCHSIA_DIR": checkout.root_dir}
            ):
                api.shac.check(
                    root=checkout.root_dir / project_dir,
                    shac_path=build_results.tool("shac"),
                    revision=revision,
                    runtime_vars={
                        "fuchsia_build_dir": api.path.relpath(
                            build_results.build_dir, checkout.root_dir
                        )
                    },
                    show_repro_cmd=False,
                    # Only emit comments for findings in the triggering repo.
                    emit_comments=(
                        project_dir == gerrit_project
                        or (project_dir == "." and gerrit_project == "fuchsia")
                    ),
                    test_id_prefix=(project_dir + "/" if project_dir != "." else None),
                )


def check_licenses(api, check_licenses_tool):
    """Check the validity of licenses.

    Args:
        check_licenses_tool (Path): Path to the check-licenses tool.

    Raises:
        StepFailure: One or more errors found in licenses.
    """
    check_licenses_command = [
        check_licenses_tool,
        "-output_license_file=false",
    ]

    step = api.step(
        "check licenses",
        check_licenses_command,
        stderr=api.raw_io.output_text(),
        ok_ret="any",
    )

    try:
        step.presentation.logs["stderr"] = step.stderr
        if step.retcode:
            step.presentation.status = api.step.FAILURE
            raise api.step.StepFailure(
                api.buildbucket_util.summary_message(
                    step.stderr,
                    "(failure summary truncated, see `check licenses` stderr for "
                    "full failure details)",
                )
            )
    finally:
        api.reported_step.upload("check_licenses", step)


def check_metadata(api, checkout):
    """Check the validity of METADATA.textproto files.

    Raises:
        StepFailure: One of more errors found in METADATA.textproto files.
    """
    with api.step.nest("check metadata validity"):
        if api.buildbucket_util.is_tryjob:
            files = checkout.changed_files(base=None)
        else:
            files = checkout.list_files(
                file="*METADATA.textproto*",
            )
        files = [str(api.path.relpath(f, checkout.root_dir)) for f in files]
        files = [
            f
            for f in files
            if f.endswith(("METADATA.textproto")) and "third_party" not in f
        ]

        malformed_files = []
        for f in files:
            # read_text expects an absolute path.
            txt = api.file.read_text(
                f"read {f}",
                checkout.root_dir / f,
            )
            try:
                api.proto.decode(txt, Metadata, "TEXTPB")
            except Exception:
                malformed_files.append(f)

        if malformed_files:
            format_step = api.step.empty("METADATA.textproto")
            format_step.presentation.status = api.step.FAILURE

            header = f"{pluralize('file', len(malformed_files))} not formatted: \n"
            error_lines = [header] + [
                f"- {api.path.relpath(f, checkout.root_dir)}" for f in malformed_files
            ]

            api.reported_step.upload("metadata_validity", format_step)
            raise api.step.StepFailure(
                api.buildbucket_util.summary_message(
                    "\n".join(error_lines), "(truncated)"
                )
            )


def GenTests(api):
    source_info = [
        {
            "name": "integration",
            "remote": "https://fuchsia.googlesource.com/integration",
            "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738",
            "relativePath": "integration",
        },
        {
            "name": "fuchsia",
            "remote": "https://fuchsia.googlesource.com/fuchsia",
            "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738",
            "relativePath": ".",
        },
    ]

    def props(**kwargs):
        return api.properties(
            manifest="fuchsia",
            remote="https://fuchsia.googlesource.com/integration",
            fint_params_path="specs/static-checks.textproto",
            incremental=True,
            **kwargs,
        )

    yield (
        api.buildbucket_util.test("default_ci", tryjob=False)
        + props()
        + api.checkout.source_info(source_info)
    )

    yield (
        api.buildbucket_util.test("default_cq", tryjob=True)
        + props(additional_shac_project_dirs=["vendor/foo"])
        + api.checkout.source_info(source_info)
    )

    yield (
        api.buildbucket_util.test("failed_build", tryjob=True, status="FAILURE")
        + props()
        + api.checkout.source_info(source_info)
        + api.step_data("build.ninja", retcode=1)
    )

    yield (
        api.buildbucket_util.test("failed_shac", tryjob=True, status="FAILURE")
        + props()
        + api.checkout.source_info(source_info)
        + api.step_data(
            "run shac checks.shac check",
            retcode=2,
        )
    )

    yield (
        api.buildbucket_util.test(
            "failed_licenses_check", tryjob=True, status="FAILURE"
        )
        + props()
        + api.checkout.source_info(source_info)
        + api.step_data(
            "check licenses",
            stderr=api.raw_io.output_text("Encountered prohibited license types."),
            retcode=1,
        )
    )

    yield (
        api.buildbucket_util.test(
            "failed_metadata_textproto_check", tryjob=True, status="FAILURE"
        )
        + props()
        + api.checkout.source_info(source_info)
        # TODO(olivernewman): This step name is referring to the step that seeds
        # the cache.  This is confusing as this step should be able to have its
        # own step that isn't referring to a different step.
        + api.git.get_changed_files(
            "check metadata validity.get changed files.git diff-tree",
            ["bar/METADATA.textproto"],
        )
        + api.step_data(
            "check metadata validity.read fuchsia/bar/METADATA.textproto",
            api.file.read_text("malformed proto"),
        )
    )