| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from google.protobuf import json_format as jsonpb |
| from google.protobuf import struct_pb2 |
| |
| from recipe_engine import post_process |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2 |
| from PB.recipes.fuchsia.fuchsia.spec import Fuchsia |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/checkout", |
| "fuchsia/jiri", |
| "fuchsia/release", |
| "recipe_engine/assertions", |
| "recipe_engine/file", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| ] |
| |
| PROPERTIES = { |
| "project": Property( |
| kind=str, |
| help="The name of the project", |
| default="integration", |
| ), |
| "remote": Property( |
| kind=str, |
| help="The remote URL of the manifest repo", |
| default="https://fuchsia.googlesource.com/integration", |
| ), |
| "manifest": Property( |
| kind=str, |
| help="The name of the manifest to import from the integration repo", |
| default="minimal", |
| ), |
| "is_release_version": Property( |
| kind=bool, help="Whether or not checkout is a release version", default=False |
| ), |
| "attributes": Property( |
| kind=List(str), |
| help="Attributes corresponding to optional packages or projects to checkout", |
| default=(), |
| ), |
| "use_incremental_cache": Property( |
| kind=bool, |
| help="Whether to restore an old checkout from the cache", |
| default=False, |
| ), |
| "allow_skipping_patch": Property( |
| kind=bool, |
| help="Whether to allow skipping the patch if the project doesn't exist", |
| default=False, |
| ), |
| "enable_submodules": Property( |
| kind=bool, |
| help="Whether to enable submodules in the checkout", |
| default=False, |
| ), |
| } |
| |
| |
| def RunSteps( |
| api, |
| project, |
| remote, |
| manifest, |
| is_release_version, |
| attributes, |
| use_incremental_cache, |
| allow_skipping_patch, |
| enable_submodules, |
| ): |
| checkout_spec = Fuchsia.Checkout( |
| project=project, |
| remote=remote, |
| manifest=manifest, |
| is_release_version=is_release_version, |
| attributes=attributes, |
| allow_skipping_patch=allow_skipping_patch, |
| enable_submodules=enable_submodules, |
| ) |
| checkout = api.checkout.from_spec( |
| checkout_spec, |
| use_incremental_cache=use_incremental_cache, |
| ) |
| assert checkout.root_dir |
| assert checkout.snapshot_file |
| checkout.integration_revision # pylint: disable=pointless-statement |
| checkout.release_branch # pylint: disable=pointless-statement |
| checkout.release_version # pylint: disable=pointless-statement |
| checkout.list_files() |
| checkout.changed_files(base=None) |
| checkout.upload_results("artifact_gcs_bucket") |
| checkout.check_clean() |
| |
| # Checkouts should not be allowed to be rooted in start_dir. |
| with api.assertions.assertRaises(ValueError): |
| api.checkout.with_options( |
| manifest=manifest, remote=remote, path=api.path.start_dir |
| ) |
| |
| # manifest and remote are required. |
| with api.assertions.assertRaises(AssertionError): |
| api.checkout.with_options(manifest="", remote="") |
| |
| |
| def GenTests(api): |
| source_info = [ |
| { |
| "name": "integration", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738", |
| "relativePath": "integration", |
| }, |
| ] |
| |
| yield ( |
| api.buildbucket_util.test("default", tryjob=True) |
| + api.checkout.source_info(source_info) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "global_integration_commit_with_change", |
| tryjob=True, |
| revision="base-revision", |
| repo="integration", |
| ) |
| + api.properties( |
| **{ |
| "$fuchsia/checkout": { |
| "respect_gitiles_commit_with_gerrit_change": True, |
| "cherry_pick_patches": True, |
| } |
| } |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "global_integration_tryjob", tryjob=True, repo="integration" |
| ) |
| + api.checkout.source_info(source_info) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "smart_integration_tryjob_on_integration_patch", |
| tryjob=True, |
| repo="integration", |
| properties=jsonpb.ParseDict( |
| { |
| "recipes_integration_ref_mapping": { |
| "refs/heads/original": "refs/heads/mapped", |
| }, |
| }, |
| struct_pb2.Struct(), |
| ), |
| ) |
| + api.checkout.source_info( |
| [ |
| { |
| "name": "smart-integration", |
| "remote": "https://fuchsia.googlesource.com/smart-integration", |
| "revision": "a491082dc1b632bbcd60ba3618d20b503c2de738", |
| "relativePath": "integration", |
| }, |
| ] |
| ) |
| + api.step_data( |
| "checkout.get change details", |
| api.json.output( |
| { |
| "branch": "original", |
| "revisions": { |
| "d4e5f6": {"_number": 7, "ref": "refs/changes/00/100/7"}, |
| }, |
| } |
| ), |
| ) |
| ) |
| |
| # The current patchset may not immediately be queryable in Gerrit if it was |
| # recently uploaded, due to replication delays. If it's not included in the |
| # change details, we should retry until it is included. |
| yield ( |
| api.buildbucket_util.test("gerrit_replication_delay", tryjob=True, patch_set=2) |
| + api.checkout.source_info(source_info) |
| + api.step_data( |
| "checkout.get change details", |
| api.json.output( |
| { |
| "branch": "main", |
| "revisions": { |
| "d4e5f6": {"_number": 1, "ref": "refs/changes/00/100/1"}, |
| }, |
| } |
| ), |
| ) |
| + api.step_data( |
| "checkout.get change details (2)", |
| api.json.output( |
| { |
| "branch": "main", |
| "revisions": { |
| "d4e5f6": {"_number": 1, "ref": "refs/changes/00/100/1"}, |
| "a1b2c3": {"_number": 2, "ref": "refs/changes/00/100/2"}, |
| }, |
| } |
| ), |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "global_integration_release_ci", |
| tryjob=False, |
| repo="integration", |
| git_ref="refs/heads/releases/foo", |
| ) |
| + api.properties( |
| is_release_version=True, |
| attributes=["attr1", "attr2"], |
| ) |
| + api.checkout.source_info(source_info) |
| + api.release.ref_to_release_version( |
| "releases/0.20190603.0.1", |
| nesting="checkout.resolve release version", |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "global_integration_release_ci_invalid_branch", |
| tryjob=False, |
| repo="integration", |
| git_ref="refs/heads/NOT-A-RELEASE-BRANCH", |
| ) |
| + api.properties(is_release_version=True) |
| + api.checkout.source_info(source_info) |
| + api.release.ref_to_release_version( |
| "releases/0.20190603.0.1", |
| nesting="checkout.resolve release version", |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "fuchsia_enable_submodules", |
| tryjob=False, |
| repo="integration", |
| ) |
| + api.checkout.source_info(source_info) |
| + api.properties(enable_submodules=True) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "local_integration_tryjob_patch_no_such_project", |
| tryjob=True, |
| repo="fuchsia", |
| status="FAILURE", |
| ) |
| + api.step_data("checkout.jiri project", api.json.output([])) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "local_integration_ci", tryjob=False, repo="not-fuchsia" |
| ) |
| + api.checkout.source_info(source_info) |
| ) |
| |
| # By default, `api.file.read_text()` will return an empty string rather than |
| # failing when run in recipe unit-testing mode. So make sure that we cover |
| # the production code path, in which `read_text()` will raise an exception |
| # rather than returning an empty string. |
| yield ( |
| api.buildbucket_util.test("missing_patchfile", tryjob=True) |
| + api.checkout.source_info(source_info) |
| + api.step_data("checkout.read patches.json", api.file.errno("ENOENT")) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("tryjob_with_patchfile", tryjob=True, repo="fuchsia") |
| + api.checkout.source_info(source_info) |
| + api.checkout.patchfile( |
| [ |
| { |
| "ref": "refs/changes/33/112233/1", |
| "host": "example-review.googlesource.com", |
| "project": "not_fuchsia", |
| } |
| ] |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("with_package_overrides", tryjob=True) |
| + api.step_data( |
| "checkout.read package overrides", |
| api.file.read_json({"path/to/package": "abcdef"}), |
| ) |
| + api.checkout.source_info(source_info) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "fail_to_patch_over_gerrit_change", |
| tryjob=True, |
| repo="fuchsia", |
| status="FAILURE", |
| ) |
| + api.checkout.patchfile( |
| [ |
| { |
| "ref": "refs/changes/33/112233/1", |
| "host": "fuchsia-review.googlesource.com", |
| "project": "fuchsia", |
| } |
| ] |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "fail_to_patch_same_project_many_times", |
| tryjob=True, |
| repo="fuchsia", |
| status="FAILURE", |
| ) |
| + api.checkout.patchfile( |
| [ |
| { |
| "ref": "refs/changes/33/112233/1", |
| "host": "fuchsia-review.googlesource.com", |
| "project": "zircon", |
| }, |
| { |
| "ref": "refs/changes/66/445566/2", |
| "host": "fuchsia-review.googlesource.com", |
| "project": "zircon", |
| }, |
| ] |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "patch_json_third_party", |
| tryjob=True, |
| repo="integration", |
| status="SUCCESS", |
| ) |
| + api.properties(enable_submodules=True) |
| + api.step_data( |
| "checkout.jiri project submodule check patchfile", |
| api.json.output( |
| [ |
| { |
| "name": "libc-tests", |
| "relativePath": "third_party/libc-tests", |
| "gitsubmoduleof": "fuchsia", |
| "revision": "aaaaaa", |
| }, |
| { |
| "name": "fuchsia", |
| "relativePath": ".", |
| "revision": "bbbbbb", |
| }, |
| ] |
| ), |
| ) |
| + api.checkout.patchfile( |
| [ |
| { |
| "ref": "refs/changes/33/112233/1", |
| "host": "fuchsia-review.googlesource.com", |
| "project": "fuchsia", |
| }, |
| { |
| "ref": "refs/changes/66/445566/2", |
| "host": "fuchsia-review.googlesource.com", |
| "project": "libc-tests", |
| }, |
| ] |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "patch_third_party_change", |
| tryjob=True, |
| repo="libc-tests", |
| status="SUCCESS", |
| ) |
| + api.properties(enable_submodules=True) |
| + api.checkout.source_info(source_info) |
| + api.step_data( |
| "checkout.jiri project submodule check patch", |
| api.json.output( |
| [ |
| { |
| "name": "libc-tests", |
| "relativePath": "third_party/libc-tests", |
| "gitsubmoduleof": "fuchsia", |
| "revision": "aaaaaa", |
| }, |
| { |
| "name": "fuchsia", |
| "relativePath": ".", |
| "revision": "bbbbbb", |
| }, |
| ] |
| ), |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "patch_integration_change", |
| tryjob=True, |
| repo="integration", |
| status="SUCCESS", |
| ) |
| + api.properties(enable_submodules=True) |
| + api.checkout.source_info(source_info) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "when_patchfile_host_has_protocol", tryjob=True, repo="fuchsia" |
| ) |
| + api.checkout.source_info(source_info) |
| + api.checkout.patchfile( |
| [ |
| { |
| "ref": "refs/changes/33/112233/1", |
| "host": "https://fuchsia-review.googlesource.com", |
| "project": "foo", |
| } |
| ] |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "should_ignore_patches_for_non_dependant_projects", |
| tryjob=True, |
| repo="fuchsia", |
| ) |
| + api.checkout.source_info(source_info) |
| + api.checkout.patchfile( |
| [ |
| { |
| "ref": "refs/changes/33/112233/1", |
| "host": "https://fuchsia-review.googlesource.com", |
| "project": "foo", |
| } |
| ] |
| ) |
| + api.step_data("checkout.jiri project foo", api.json.output([])) |
| ) |
| |
| # We want to be able to run tryjobs on integration CLs to test property |
| # changes, which take effect via recipe_wrapper even when the integration |
| # repo isn't included in the checkout. |
| yield ( |
| api.buildbucket_util.test( |
| "ignore_patch_for_nonexistent_integration_project", |
| tryjob=True, |
| repo="integration", |
| ) |
| + api.checkout.source_info(source_info) |
| + api.step_data("checkout.jiri project", api.json.output([])) |
| + api.post_process(post_process.MustRun, "checkout.skipping patch") |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "allow_skipping_patch", |
| tryjob=True, |
| repo="doesnotexist", |
| ) |
| + api.properties(allow_skipping_patch=True) |
| + api.checkout.source_info(source_info) |
| + api.step_data("checkout.jiri project", api.json.output([])) |
| + api.post_process(post_process.MustRun, "checkout.skipping patch") |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "should_ignore_recipe_changes", tryjob=True, repo="infra/recipes" |
| ) |
| + api.checkout.source_info(source_info) |
| + api.step_data("get changed files.jiri project", api.json.output([])) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("sso_manifest_remote", repo="not-fuchsia") |
| + api.properties(remote="sso://manifest-host/manifest") |
| + api.checkout.source_info(source_info) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("use_gitiles_commit_from_properties", tryjob=False) |
| + api.properties( |
| **{ |
| "$fuchsia/checkout": { |
| "gitiles_commit": jsonpb.MessageToDict( |
| common_pb2.GitilesCommit( |
| host="fuchsia.googlesource.com", |
| project="foo", |
| id="deadbeef", |
| ) |
| ) |
| } |
| } |
| ) |
| + api.checkout.source_info(source_info) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("use_incremental_cache", tryjob=False) |
| + api.properties(use_incremental_cache=True) |
| + api.checkout.source_info(source_info) |
| ) |
| |
| # No gerrit_changes or gitiles_commit. |
| yield api.test("empty_build_input") + api.checkout.source_info(source_info) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "local_integration_tryjob_patch_submodule", |
| tryjob=True, |
| repo="third_party/github.com/foobar/foobar", |
| status="SUCCESS", |
| ) |
| + api.checkout.source_info(source_info) |
| + api.step_data( |
| "checkout.jiri patch", retcode=api.jiri.NoSuchProjectError.EXIT_CODE |
| ) |
| + api.step_data("checkout.jiri project", api.json.output([])) |
| + api.step_data( |
| "checkout.list submodules", |
| api.raw_io.stream_output_text( |
| "\n".join( |
| [ |
| "submodule.third_party/foobar.path=third_party/foobar", |
| "submodule.third_party/foobar.url=https://fuchsia.googlesource.com/third_party/github.com/foobar/foobar", |
| "submodule.third_party/foobar.name=third_party/foobar", |
| "submodule.third_party/foobar.ignore=all", |
| "submodule.build/secondary/third_party/foobaz.path=build/secondary/third_party/foobaz", |
| "submodule.build/secondary/third_party/foobaz.url=https://fuchsia.googlesource.com/foobaz-gn", |
| "submodule.build/secondary/third_party/foobaz.name=foobaz-gn", |
| "submodule.build/secondary/third_party/foobaz.ignore=all", |
| ] |
| ) |
| ), |
| ) |
| ) |
