| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Recipe for checks that require a checkout but not a full build.""" |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| from RECIPE_MODULES.fuchsia.utils import pluralize |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| DEPS = [ |
| "fuchsia/autocorrelator", |
| "fuchsia/build", |
| "fuchsia/buildbucket_util", |
| "fuchsia/checkout", |
| "fuchsia/git", |
| "fuchsia/utils", |
| "recipe_engine/context", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "manifest": Property(kind=str, help="Jiri manifest to use"), |
| "remote": Property(kind=str, help="Remote manifest repository"), |
| "check_licenses_configs": Property( |
| kind=List(str), |
| help="Config file paths to pass to check-licenses, relative to checkout root", |
| default=(), |
| ), |
| "fint_params_path": Property(kind=str, help="Spec path to pass to fint"), |
| } |
| |
| |
| def RunSteps(api, remote, manifest, check_licenses_configs, fint_params_path): |
| checkout = api.checkout.fuchsia_with_options( |
| manifest=manifest, |
| remote=remote, |
| ) |
| |
| try: |
| build_results = api.build.with_options( |
| checkout=checkout, fint_params_path=fint_params_path |
| ) |
| with api.context(cwd=checkout.root_dir): |
| check_docs(api, build_results.tool("doc-checker")) |
| check_gn_formatting(api, checkout, build_results.tool("gn")) |
| check_licenses( |
| api, build_results.tool("check-licenses"), check_licenses_configs |
| ) |
| except api.step.StepFailure as exc: |
| if api.buildbucket_util.is_tryjob: |
| with api.step.nest("check for correlated failures") as parent_step: |
| api.autocorrelator.check_try( |
| "check try", |
| exc, |
| exc.reason, |
| ) |
| api.autocorrelator.check_ci( |
| "check ci", |
| checkout.integration_revision, |
| exc, |
| exc.reason, |
| ) |
| api.autocorrelator.set_properties(parent_step) |
| raise api.autocorrelator.compose_exception(exc) |
| |
| |
| def check_docs(api, doc_checker_tool): |
| """Verify structure and content of documentation files. |
| |
| Args: |
| doc_checker_tool (Path): Path to doc-checker tool. |
| |
| Raises: |
| StepFailure: One or more errors found in documentation. |
| """ |
| step = api.step( |
| "doc check", |
| [doc_checker_tool, "--local-links-only"], |
| stdout=api.raw_io.output_text(), |
| ok_ret="any", |
| ) |
| step.presentation.logs["stdout"] = step.stdout |
| if step.retcode: |
| step.presentation.status = api.step.FAILURE |
| raise api.step.StepFailure( |
| api.buildbucket_util.summary_message( |
| step.stdout, |
| "(failure summary truncated, see `doc check` stdout for full " |
| "failure details)", |
| ) |
| ) |
| |
| |
| def check_gn_formatting(api, checkout, gn_tool): |
| """Check gn formatting. |
| |
| Does not attempt to suggest fixes; the tricium recipe does that. |
| |
| Args: |
| checkout (CheckoutResults): CheckoutResults object. |
| gn_tool (Path): Path to gn tool. |
| |
| Raises: |
| StepFailure: One or more GN files are not formatted properly. |
| """ |
| with api.step.nest("gn format"): |
| # On try jobs, only check the modified files. On CI, check them all. |
| if api.buildbucket_util.is_tryjob: |
| files = checkout.changed_files( |
| test_data=["bar/BUILD.gn", "foo.cc", "third_party/foo/BUILD.gn"], |
| deleted=False, |
| ) |
| else: |
| step_result = api.git.ls_files( |
| step_name="get gn files", |
| file="*.gn*", |
| test_data="bar/BUILD.gn\nfoo.cc\nthird_party/foo/BUILD.gn", |
| ) |
| files = step_result.stdout.strip("\n").split("\n") |
| files = [ |
| f for f in files if f.endswith((".gn", ".gni")) and "third_party" not in f |
| ] |
| if files: |
| format_step = api.step( |
| "gn format --dry-run", |
| [gn_tool, "format", "--dry-run"] + files, |
| stdout=api.raw_io.output_text(), |
| ok_ret=(0, 2), |
| ) |
| format_step.presentation.logs["stdout"] = format_step.stdout |
| # Retcode 2 indicates that formatting is successful but differs |
| # from disk. See `gn help format` for more information. |
| if format_step.retcode == 2: |
| format_step.presentation.status = api.step.FAILURE |
| unformatted_files = format_step.stdout.strip("\n").split("\n") |
| header = "%s not formatted: \n" % pluralize( |
| "file", len(unformatted_files) |
| ) |
| error_lines = [header] + [ |
| "- %s" % api.path.relpath(f, checkout.root_dir) |
| for f in unformatted_files |
| ] |
| raise api.step.StepFailure( |
| api.buildbucket_util.summary_message( |
| "\n".join(error_lines), |
| "(failure summary truncated, see `gn format --dry-run` " |
| "stdout for full failure details)", |
| escape_markdown=False, |
| ) |
| ) |
| |
| |
| def check_licenses(api, check_licenses_tool, check_licenses_configs): |
| """Check the validity of licenses. |
| |
| Args: |
| check_licenses_tool (Path): Path to gn tool. |
| check_licenses_configs (seq(str)): See corresponding recipe property |
| docstring. |
| |
| Raises: |
| StepFailure: One or more errors found in licenses. |
| """ |
| check_licenses_command = [ |
| check_licenses_tool, |
| "-output_license_file=false", |
| ] |
| if check_licenses_configs: |
| check_licenses_command.append( |
| "-config_file=%s" % ",".join(check_licenses_configs) |
| ) |
| step = api.step( |
| "check licenses", |
| check_licenses_command, |
| stderr=api.raw_io.output_text(), |
| ok_ret="any", |
| ) |
| step.presentation.logs["stderr"] = step.stderr |
| if step.retcode: |
| step.presentation.status = api.step.FAILURE |
| raise api.step.StepFailure( |
| api.buildbucket_util.summary_message( |
| step.stderr, |
| "(failure summary truncated, see `check licenses` stderr for " |
| "full failure details)", |
| ) |
| ) |
| |
| |
| def GenTests(api): |
| source_info = [ |
| { |
| "name": "integration", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738", |
| "relativePath": "integration", |
| }, |
| { |
| "name": "fuchsia", |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738", |
| "relativePath": ".", |
| }, |
| ] |
| |
| def props(**kwargs): |
| return api.properties( |
| manifest="fuchsia", |
| remote="https://fuchsia.googlesource.com/integration", |
| fint_params_path="specs/static-checks.textproto", |
| **kwargs |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("default_ci", tryjob=False) |
| + props(check_licenses_configs=["config1.json", "config2.json"]) |
| ) |
| |
| yield api.buildbucket_util.test("default_cq", tryjob=True) + props() |
| |
| autocorrelator_props = { |
| "$fuchsia/autocorrelator": {"ci_bucket": "ci", "ci_builder": "builder"} |
| } |
| |
| def autocorrelator_test_data(): |
| return api.autocorrelator.check_try( |
| "check for correlated failures.check try", |
| test_data=[{"build_id": "456", "score": 0.98, "is_green": False}], |
| ) + api.autocorrelator.check_ci( |
| "check for correlated failures.check ci", |
| test_data={ |
| "build_id": "789", |
| "score": 0.96, |
| "is_green": False, |
| "commit_dist": 0, |
| }, |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("failed_build", tryjob=True, status="failure") |
| + api.checkout.source_info(source_info) |
| + api.step_data("build.ninja", retcode=1) |
| + autocorrelator_test_data() |
| + props(**autocorrelator_props) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("failed_doc_check", tryjob=True, status="failure") |
| + api.checkout.source_info(source_info) |
| + api.step_data( |
| "doc check", api.raw_io.stream_output_text("Found 2 errors."), retcode=1 |
| ) |
| + autocorrelator_test_data() |
| + props(**autocorrelator_props) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("failed_gn_format", tryjob=True, status="failure") |
| + api.checkout.source_info(source_info) |
| + api.step_data( |
| "gn format.gn format --dry-run", |
| api.raw_io.stream_output_text("foo/BUILD.gn\nbar/BUILD.gn\n"), |
| retcode=2, |
| ) |
| + autocorrelator_test_data() |
| + props(**autocorrelator_props) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "failed_licenses_check", tryjob=True, status="failure" |
| ) |
| + api.checkout.source_info(source_info) |
| + api.step_data( |
| "check licenses", |
| stderr=api.raw_io.output_text("Encountered prohibited license types."), |
| retcode=1, |
| ) |
| + autocorrelator_test_data() |
| + props(**autocorrelator_props) |
| ) |