| # Copyright 2020 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for computing test owners, for use by Flake Fetcher |
| |
| Reads a test manifest from GCS that's expected to be a list of JSON objects, |
| each of which contains a test name and the test's GN label. |
| |
| Then the recipe uses a Fuchsia checkout to compute test owners and Monorail |
| components based on the nearest OWNERS file to the GN label. Finally, the |
| recipe uploads the owners data to another JSON file in GCS that Flake Fetcher |
| uses to assign bugs for flaky tests to the tests' owners and put the bugs in |
| the appropriate components. |
| """ |
| |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| "fuchsia/checkout", |
| "fuchsia/gsutil", |
| "fuchsia/status_check", |
| "recipe_engine/cipd", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "manifest": Property(kind=str, help="Jiri manifest to use"), |
| "remote": Property(kind=str, help="Remote manifest repository"), |
| "owners_gcs_bucket": Property(kind=str, help="GCS bucket to upload test owners to"), |
| "owners_gcs_path": Property( |
| kind=str, |
| help="Path within `owners_gcs_bucket` to upload the test owners manifest to", |
| ), |
| "dry_run": Property(kind=bool, help="Don't upload results to GCS", default=False), |
| } |
| |
| BIGQUERY_PROJECT = "fuchsia-infra" |
| |
| # Queries ResultDB to find all unique "test_id" and "gn_label" combinations. |
| TEST_NAMES_QUERY = """ |
| SELECT DISTINCT |
| test_id AS test_name, |
| SPLIT(tags.value, ':')[OFFSET(0)] AS gn_label |
| FROM |
| `fuchsia-infra.resultdb.try`, |
| UNNEST(tags) AS tags |
| WHERE |
| partition_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 7 day) |
| AND tags.key = 'gn_label' |
| AND tags.value != '' |
| UNION DISTINCT |
| SELECT DISTINCT |
| test_id AS test_name, |
| SPLIT(tags.value, ':')[OFFSET(0)] AS gn_label |
| FROM |
| `fuchsia-infra.resultdb.ci`, |
| UNNEST(tags) AS tags |
| WHERE |
| partition_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 7 day) |
| AND tags.key = 'gn_label' |
| AND tags.value != '' |
| """ |
| |
| |
| def RunSteps(api, manifest, remote, owners_gcs_bucket, owners_gcs_path, dry_run): |
| checkout_dir = api.path["start_dir"].join("checkout") |
| api.checkout.with_options( |
| path=checkout_dir, |
| manifest=manifest, |
| remote=remote, |
| # No need to fetch CIPD packages, we only need to look at OWNERS |
| # files and those are always checked into git. |
| fetch_packages=False, |
| run_hooks=False, |
| ) |
| |
| workdir = api.path.mkdtemp("workdir") |
| test_manifest_path = workdir.join("test_manifest.json") |
| |
| bq_path = api.cipd.ensure_tool( |
| "fuchsia/infra/bigquery/${platform}", |
| "git_revision:315f9e290334a63cd265c67000069b82323d732f", |
| ) |
| api.step( |
| "query test names", |
| [ |
| bq_path, |
| "query", |
| "-project", |
| BIGQUERY_PROJECT, |
| "-input", |
| api.raw_io.input(TEST_NAMES_QUERY), |
| "-json-output", |
| test_manifest_path, |
| ], |
| ) |
| if dry_run: |
| api.file.read_text("log test_manifest.json", test_manifest_path) |
| |
| owners_path = workdir.join("test_owners.json") |
| with api.context(cwd=checkout_dir): |
| api.step( |
| "find owners", |
| [api.resource("find_owners.py"), test_manifest_path, owners_path], |
| ) |
| |
| if dry_run: |
| api.file.read_text("log test_owners.json", owners_path) |
| else: |
| api.gsutil.upload( |
| name="upload test owners", |
| src=owners_path, |
| bucket=owners_gcs_bucket, |
| dst=owners_gcs_path, |
| ) |
| |
| |
| def GenTests(api): |
| def properties(**kwargs): |
| props = { |
| "manifest": "fuchsia/flower", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "owners_gcs_bucket": "owners", |
| "owners_gcs_path": "test_owners.json", |
| } |
| props.update(kwargs) |
| return api.properties(**props) |
| |
| yield api.status_check.test("basic") + properties() |
| |
| yield api.status_check.test("dry_run") + properties(dry_run=True) |