| # Copyright 2020 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for computing test owners, for use by Flake Fetcher |
| |
| Reads a test manifest from GCS that's expected to be a list of JSON objects, |
| each of which contains a test name and the test's GN label. |
| |
| Then the recipe uses a Fuchsia checkout to compute test owners and Issue Tracker |
| components based on the nearest OWNERS file to the GN label. Finally, the recipe |
| uploads the owners data to another JSON file in GCS that Flake Fetcher uses to |
| assign bugs for flaky tests to the tests' owners and put the bugs in the |
| appropriate components. |
| """ |
| |
| import datetime |
| |
| from PB.recipes.fuchsia.test_owners import InputProperties |
| |
| DEPS = [ |
| "fuchsia/bigquery", |
| "fuchsia/bqupload", |
| "fuchsia/checkout", |
| "fuchsia/gsutil", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| "recipe_engine/time", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| BIGQUERY_PROJECT = "fuchsia-infra" |
| |
| # Queries ResultDB to find all unique "test_id" and "gn_label" combinations. |
| TEST_NAMES_QUERY = """ |
| SELECT DISTINCT |
| test_id AS test_name, |
| SPLIT(tags.value, ':')[OFFSET(0)] AS gn_label |
| FROM |
| `fuchsia-infra.resultdb.try`, |
| UNNEST(tags) AS tags |
| WHERE |
| partition_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 7 day) |
| AND tags.key = 'gn_label' |
| AND tags.value != '' |
| UNION DISTINCT |
| SELECT DISTINCT |
| test_id AS test_name, |
| SPLIT(tags.value, ':')[OFFSET(0)] AS gn_label |
| FROM |
| `fuchsia-infra.resultdb.ci`, |
| UNNEST(tags) AS tags |
| WHERE |
| partition_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 7 day) |
| AND tags.key = 'gn_label' |
| AND tags.value != '' |
| """ |
| |
| |
| def RunSteps(api, props): |
| checkout_dir = api.checkout.with_options( |
| manifest=props.manifest, |
| remote=props.remote, |
| # No need to fetch CIPD packages, we only need to look at OWNERS |
| # files and those are always checked into git. |
| fetch_packages=False, |
| ) |
| |
| workdir = api.path.mkdtemp("workdir") |
| test_manifest_path = workdir.join("test_manifest.json") |
| api.bigquery.project = BIGQUERY_PROJECT |
| api.bigquery.query( |
| "query test names", |
| TEST_NAMES_QUERY, |
| test_manifest_path, |
| ) |
| if props.dry_run: |
| api.file.read_text("log test_manifest.json", test_manifest_path) |
| |
| owners_path = workdir.join("test_owners.json") |
| with api.context(cwd=checkout_dir): |
| api.step( |
| "find owners", |
| [api.resource("find_owners.py"), test_manifest_path, owners_path], |
| timeout=datetime.timedelta(minutes=5), |
| ) |
| |
| tests = api.file.read_json( |
| "read test_owners.json", |
| owners_path, |
| test_data=[ |
| { |
| "test_name": "test1", |
| "owners": ["x@gmail.com", "y@gmail.com"], |
| }, |
| { |
| "test_name": "test2", |
| "owners": ["x@gmail.com", "z@gmail.com"], |
| }, |
| ], |
| ) |
| |
| date = str(api.time.utcnow().date()) # E.g. "2021-05-27". |
| bq_rows = [] |
| # The BQ table has a slightly different schema from that of |
| # test_owners.json; it has one row per (test, owner) pair. |
| for test in tests: |
| for owner in test.pop("owners"): |
| row = {"owner": owner, "date_created": date} |
| row.update(test) |
| bq_rows.append(row) |
| |
| if not props.dry_run: |
| api.gsutil.upload( |
| name="upload to GCS", |
| src=owners_path, |
| bucket=props.owners_gcs_bucket, |
| dst=props.owners_gcs_path, |
| ) |
| api.bqupload.insert( |
| "upload to bigquery", |
| project="fuchsia-infra", |
| dataset="test_owners", |
| table="test_owners", |
| rows=bq_rows, |
| ) |
| |
| |
| def GenTests(api): |
| def properties(**kwargs): |
| props = { |
| "manifest": "fuchsia/flower", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "owners_gcs_bucket": "owners", |
| "owners_gcs_path": "test_owners.json", |
| } |
| props.update(kwargs) |
| return api.properties(**props) |
| |
| yield api.test("basic") + properties() |
| |
| yield api.test("dry_run") + properties(dry_run=True) |