| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Recipe for checks that require a checkout but not a full build.""" |
| |
| |
| from RECIPE_MODULES.fuchsia.utils import pluralize |
| from PB.recipes.fuchsia.static_checks import InputProperties |
| from PB.infra.metadata import Metadata |
| |
| DEPS = [ |
| "fuchsia/autocorrelator", |
| "fuchsia/build", |
| "fuchsia/buildbucket_util", |
| "fuchsia/checkout", |
| "fuchsia/git", |
| "fuchsia/reported_step", |
| "fuchsia/shac", |
| "fuchsia/utils", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/proto", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps(api, props): |
| checkout = api.checkout.fuchsia_with_options( |
| manifest=props.manifest, |
| remote=props.remote, |
| use_incremental_cache=props.incremental_build, |
| enable_submodules=props.enable_submodules, |
| ) |
| |
| revision = checkout.integration_revision |
| gerrit_project = None |
| if changes := api.buildbucket.build.input.gerrit_changes: |
| gerrit_project = changes[0].project |
| |
| with api.autocorrelator.context(ci_base_commit=revision): |
| build_results = api.build.with_options( |
| checkout=checkout, |
| fint_params_path=props.fint_params_path, |
| incremental=props.incremental_build, |
| ) |
| with api.context(cwd=checkout.root_dir): |
| check_licenses(api, build_results.tool("check-licenses")) |
| check_metadata(api, checkout) |
| project_dirs = ["."] + list(props.additional_shac_project_dirs) |
| for project_dir in project_dirs: |
| step_name = "run shac checks" |
| if project_dir != ".": |
| step_name += f" in {project_dir}" |
| with api.step.nest(step_name), api.context( |
| # Checks in sub-repos depend on the $FUCHSIA_DIR env var to |
| # resolve the fuchsia checkout root so it can be passed through |
| # the filesystem sandbox, which would normally prevent checks in |
| # one repository from accessing files in parent directories. |
| env={"FUCHSIA_DIR": checkout.root_dir} |
| ): |
| api.shac.check( |
| root=checkout.root_dir / project_dir, |
| shac_path=build_results.tool("shac"), |
| revision=revision, |
| runtime_vars={ |
| "fuchsia_build_dir": api.path.relpath( |
| build_results.build_dir, checkout.root_dir |
| ) |
| }, |
| show_repro_cmd=False, |
| # Only emit comments for findings in the triggering repo. |
| emit_comments=( |
| project_dir == gerrit_project |
| or (project_dir == "." and gerrit_project == "fuchsia") |
| ), |
| test_id_prefix=(project_dir + "/" if project_dir != "." else None), |
| ) |
| |
| |
| def check_licenses(api, check_licenses_tool): |
| """Check the validity of licenses. |
| |
| Args: |
| check_licenses_tool (Path): Path to the check-licenses tool. |
| |
| Raises: |
| StepFailure: One or more errors found in licenses. |
| """ |
| check_licenses_command = [ |
| check_licenses_tool, |
| "-output_license_file=false", |
| ] |
| |
| step = api.step( |
| "check licenses", |
| check_licenses_command, |
| stderr=api.raw_io.output_text(), |
| ok_ret="any", |
| ) |
| |
| try: |
| step.presentation.logs["stderr"] = step.stderr |
| if step.retcode: |
| step.presentation.status = api.step.FAILURE |
| raise api.step.StepFailure( |
| api.buildbucket_util.summary_message( |
| step.stderr, |
| "(failure summary truncated, see `check licenses` stderr for " |
| "full failure details)", |
| ) |
| ) |
| finally: |
| api.reported_step.upload("check_licenses", step) |
| |
| |
| def check_metadata(api, checkout): |
| """Check the validity of METADATA.textproto files. |
| |
| Raises: |
| StepFailure: One of more errors found in METADATA.textproto files. |
| """ |
| with api.step.nest("check metadata validity"): |
| if api.buildbucket_util.is_tryjob: |
| files = checkout.changed_files(base=None) |
| else: |
| files = checkout.list_files( |
| file="*METADATA.textproto*", |
| ) |
| files = [str(api.path.relpath(f, checkout.root_dir)) for f in files] |
| files = [ |
| f |
| for f in files |
| if f.endswith(("METADATA.textproto")) and "third_party" not in f |
| ] |
| |
| malformed_files = [] |
| for f in files: |
| # read_text expects an absolute path. |
| txt = api.file.read_text( |
| f"read {f}", |
| checkout.root_dir / f, |
| ) |
| try: |
| api.proto.decode(txt, Metadata, "TEXTPB") |
| except Exception: |
| malformed_files.append(f) |
| |
| if malformed_files: |
| format_step = api.step.empty("METADATA.textproto") |
| format_step.presentation.status = api.step.FAILURE |
| |
| header = f"{pluralize('file', len(malformed_files))} not formatted: \n" |
| error_lines = [header] + [ |
| f"- {api.path.relpath(f, checkout.root_dir)}" for f in malformed_files |
| ] |
| |
| api.reported_step.upload("metadata_validity", format_step) |
| raise api.step.StepFailure( |
| api.buildbucket_util.summary_message( |
| "\n".join(error_lines), "(truncated)" |
| ) |
| ) |
| |
| |
| def GenTests(api): |
| source_info = [ |
| { |
| "name": "integration", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738", |
| "relativePath": "integration", |
| }, |
| { |
| "name": "fuchsia", |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738", |
| "relativePath": ".", |
| }, |
| ] |
| |
| def props(**kwargs): |
| return api.properties( |
| manifest="fuchsia", |
| remote="https://fuchsia.googlesource.com/integration", |
| fint_params_path="specs/static-checks.textproto", |
| incremental=True, |
| **kwargs, |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("default_ci", tryjob=False) |
| + props() |
| + api.checkout.source_info(source_info) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("default_cq", tryjob=True) |
| + props(additional_shac_project_dirs=["vendor/foo"]) |
| + api.checkout.source_info(source_info) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("failed_build", tryjob=True, status="FAILURE") |
| + props() |
| + api.checkout.source_info(source_info) |
| + api.step_data("build.ninja", retcode=1) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("failed_shac", tryjob=True, status="FAILURE") |
| + props() |
| + api.checkout.source_info(source_info) |
| + api.step_data( |
| "run shac checks.shac check", |
| retcode=2, |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "failed_licenses_check", tryjob=True, status="FAILURE" |
| ) |
| + props() |
| + api.checkout.source_info(source_info) |
| + api.step_data( |
| "check licenses", |
| stderr=api.raw_io.output_text("Encountered prohibited license types."), |
| retcode=1, |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "failed_metadata_textproto_check", tryjob=True, status="FAILURE" |
| ) |
| + props() |
| + api.checkout.source_info(source_info) |
| # TODO(olivernewman): This step name is referring to the step that seeds |
| # the cache. This is confusing as this step should be able to have its |
| # own step that isn't referring to a different step. |
| + api.git.get_changed_files( |
| "check metadata validity.get changed files.git diff-tree", |
| ["bar/METADATA.textproto"], |
| ) |
| + api.step_data( |
| "check metadata validity.read fuchsia/bar/METADATA.textproto", |
| api.file.read_text("malformed proto"), |
| ) |
| ) |