| # Copyright 2020 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Snap an integration.git release branch. |
| |
| Branching Modes |
| - Revision: Snap the branch to a specific revision. |
| - LKGR: Snap the branch using the LKGR of a set of builders. |
| - Branch: Snap the branch to another branch. |
| |
| Options |
| - Experimental: Snap the branch using the "add track" feature. |
| See release/api.py for more details. |
| - Rollback: Enable rolling back to a previous candidate version. |
| - Dryrun: Run recipe without modifying anything remotely. |
| """ |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2 |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2 |
| from PB.recipe_engine.result import RawResult |
| from PB.recipes.fuchsia.release.snap_branch import InputProperties |
| |
| DEPS = [ |
| "fuchsia/checkout", |
| "fuchsia/git", |
| "fuchsia/git_checkout", |
| "fuchsia/jiri", |
| "fuchsia/lkg", |
| "fuchsia/release", |
| "fuchsia/sso", |
| "fuchsia/status_check", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| "recipe_engine/time", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| SNAP_MESSAGE = "{prefix} Snap to {revision}" |
| PETALS_TEST_DATA_TEMPLATE = [ |
| { |
| "name": "foo", |
| "path": "{start_dir}/local/path/to/foo", |
| "remote": "https://fuchsia.googlesource.com/foo", |
| "revision": "snapme", |
| "manifest": "local/path/to/foo/manifest", |
| }, |
| { |
| "name": "bar", |
| "path": "{start_dir}/local/path/to/bar", |
| "remote": "https://fuchsia.googlesource.com/bar", |
| # Simulate revision that does not need snapping. |
| "revision": "deadbeef", |
| "manifest": "local/path/to/bar/manifest", |
| }, |
| { |
| # This should always be skipped. |
| "name": "integration", |
| "path": "{start_dir}/local/path/to/integration", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "revision": "skipme", |
| "manifest": "local/path/to/integration/manifest", |
| }, |
| ] |
| |
| |
| def RunSteps(api, props): |
| with api.step.nest("check inputs"): |
| if not ( |
| bool(props.revision) ^ bool(props.lkg_builders) ^ bool(props.source_branch) |
| ): |
| return RawResult( |
| summary_markdown="must specify exactly one of `revision`, " |
| "`lkg_builders`, or `source_branch`", |
| status=common_pb2.FAILURE, |
| ) |
| if props.source_branch and not api.release.validate_branch(props.source_branch): |
| return RawResult( |
| summary_markdown='`source_branch` must start with "releases/"', |
| status=common_pb2.FAILURE, |
| ) |
| if not api.release.validate_branch(props.target_branch): |
| return RawResult( |
| summary_markdown='`target_branch` must start with "releases/"', |
| status=common_pb2.FAILURE, |
| ) |
| # Assert target branch exists remotely. |
| https_remote = api.sso.sso_to_https(props.remote) |
| branch_revision = api.git.get_remote_branch_head( |
| step_name="get target branch HEAD", |
| url=https_remote, |
| branch=props.target_branch, |
| ) |
| if not branch_revision: |
| return RawResult( |
| summary_markdown="target branch %s does not exist" |
| % props.target_branch, |
| status=common_pb2.FAILURE, |
| ) |
| |
| # Resolve revision from one of `revision`, `lkg_builders`, or `source_branch`. |
| with api.step.nest("resolve source revision"): |
| revision = props.revision |
| if props.lkg_builders: |
| revision = api.lkg.revision( |
| step_name="get last-known-good revision", |
| builders=props.lkg_builders, |
| test_data="lkgrevision", |
| ) |
| elif props.source_branch: |
| with api.context(infra_steps=True): |
| revision = api.git.get_remote_branch_head( |
| step_name="get source branch HEAD", |
| url=https_remote, |
| branch=props.source_branch, |
| ) |
| if revision == branch_revision: |
| return RawResult( |
| summary_markdown="resolved snap revision %s is already current " |
| "revision on %s" % (revision, props.target_branch), |
| status=common_pb2.FAILURE, |
| ) |
| |
| checkout = api.checkout.fuchsia_with_options( |
| build_input=build_pb2.Build.Input( |
| gitiles_commit=common_pb2.GitilesCommit( |
| id=revision, |
| project=props.project, |
| ), |
| ), |
| project=props.project, |
| manifest=props.manifest, |
| remote=props.remote, |
| # Skip for faster checkout. |
| fetch_packages=False, |
| ) |
| integration_root = api.release.get_project_path(checkout.root_dir, props.project) |
| |
| with api.context(cwd=integration_root, infra_steps=True): |
| # Ensure tags are up-to-date. |
| api.git.fetch("origin", tags=True, refspec=props.target_branch) |
| |
| with api.step.nest("resolve release version"): |
| if props.rollback: |
| with api.step.nest("check rollback inputs"): |
| with api.context(infra_steps=True): |
| rollback_version = api.release.ref_to_release_version( |
| ref=revision, repo_path=integration_root |
| ) |
| branch_version = api.release.ref_to_release_version( |
| ref="origin/%s" % props.target_branch, |
| repo_path=integration_root, |
| ) |
| # Ensure the target revision is a previous candidate version of |
| # target branch's HEAD. |
| if ( |
| rollback_version.major != branch_version.major |
| or rollback_version.date != branch_version.date |
| or rollback_version.release != branch_version.release |
| or rollback_version.candidate >= branch_version.candidate |
| ): |
| return RawResult( |
| summary_markdown="cannot rollback to non-previous " |
| "candidate version: rollback version %s is not a " |
| "previous candidate version of %s's HEAD, %s" |
| % ( |
| rollback_version.tag_name, |
| props.target_branch, |
| branch_version.tag_name, |
| ), |
| status=common_pb2.FAILURE, |
| ) |
| # Resolve the new release version, which is the next candidate |
| # version on the branch. |
| release_version = api.release.get_next_candidate_version( |
| ref="origin/%s" % props.target_branch, |
| add_track=props.add_track, |
| repo_path=integration_root, |
| ) |
| else: |
| # If there are existing candidates on the target revision, then get |
| # the next candidate version. |
| release_version = api.release.get_next_candidate_version( |
| ref=revision, |
| add_track=props.add_track, |
| repo_path=integration_root, |
| ) |
| # Otherwise, get the next release version. This means that the |
| # target revision has no release history, and is almost certainly |
| # from the main branch. |
| if not release_version: |
| # Look at origin, since target branch will not have a local copy |
| # at this point. |
| release_version = api.release.get_next_release_version( |
| ref="origin/%s" % props.target_branch, |
| date=api.time.utcnow().date().strftime("%Y%m%d"), |
| repo_path=integration_root, |
| ) |
| if api.git.get_remote_tag(url=https_remote, tag=release_version.tag_name): |
| return RawResult( |
| summary_markdown="release version conflict: attempting snap " |
| "would conflict with existing tag %s, indicating source " |
| "revision %s has already moved forward" |
| % (release_version.tag_name, revision), |
| status=common_pb2.FAILURE, |
| ) |
| if props.must_match_milestone: |
| head_checkout, _ = api.git_checkout( |
| https_remote, |
| revision=revision, |
| cache=False, |
| ) |
| # If the resolved release version's milestone number is not equal |
| # to the milestone number at HEAD, do nothing. |
| with api.step.nest("check milestone match"): |
| if release_version.major != api.release.get_major_number(head_checkout): |
| return RawResult( |
| summary_markdown="skipping snap: milestone numbers do " |
| "not match", |
| status=common_pb2.SUCCESS, |
| ) |
| |
| with api.step.nest("snap integration"), api.context(infra_steps=True): |
| snap_message = SNAP_MESSAGE.format( |
| prefix=api.release._COMMIT_MESSAGE_PREFIX, |
| revision=api.release.shorten_revision(revision), |
| ) |
| # If the source revision has an associated release version, add it |
| # to the commit message. |
| source_version = api.release.ref_to_release_version( |
| ref=revision, |
| repo_path=integration_root, |
| ) |
| if source_version: |
| snap_message = "%s (equal to %s)" % (snap_message, source_version) |
| # Snap to target revision. |
| api.git.snap_branch( |
| snap_ref=revision, |
| branch=props.target_branch, |
| message=snap_message, |
| path=integration_root, |
| push=False, |
| ) |
| |
| with api.step.nest("snap petals"), api.context(infra_steps=True): |
| petal_paths = snap_petals( |
| api=api, |
| target_branch=props.target_branch, |
| checkout_root=checkout.root_dir, |
| integration_project=props.project, |
| skip_snap_projects=props.skip_snap_projects, |
| ) |
| for petal_path in petal_paths: |
| with api.context(cwd=petal_path): |
| api.git.push( |
| step_name="push commit", |
| refs="HEAD:refs/heads/%s" % props.target_branch, |
| dryrun=props.dryrun, |
| ) |
| |
| with api.context(cwd=integration_root, infra_steps=True), api.step.nest( |
| "push integration" |
| ) as presentation: |
| # Squash integration snap and petal syncs together. |
| if petal_paths: |
| api.git.add(step_name="add petal syncs", add_all=True) |
| api.git.commit(step_name="squash petal syncs", amend=True, no_edit=True) |
| release_revision = api.git.get_hash() |
| # Tag the snap commit. |
| api.git.tag(release_version.tag_name, step_name="tag release") |
| # Push commit and release version. |
| api.git.push( |
| step_name="push release", |
| refs=[ |
| "HEAD:refs/heads/%s" % props.target_branch, |
| "refs/tags/%s" % release_version.tag_name, |
| ], |
| atomic=True, |
| dryrun=props.dryrun, |
| ) |
| api.release.set_output_properties( |
| presentation, release_revision, release_version |
| ) |
| |
| if not props.dryrun and props.downstream_builders: |
| api.release.lookup_builds( |
| builders=props.downstream_builders, |
| revision=release_revision, |
| remote=https_remote, |
| ) |
| |
| |
| def snap_petals( |
| api, target_branch, checkout_root, integration_project, skip_snap_projects |
| ): |
| """Snap any existing petal branches to their global integration revisions. |
| Sync manifest revisions in global integration to their new snap revisions |
| created by the snaps. |
| |
| Args: |
| target_branch (str): Branch to snap to. |
| checkout_root (Path): Path including all petal checkouts. |
| |
| Returns: |
| list(str): Local petal git directories. |
| |
| TODO(fxbug.dev/89996): Deprecate the term "petal" since this only applies to |
| the Fuchsia flower model but not projects generically. |
| """ |
| skip_snap_projects = set(skip_snap_projects or []) |
| with api.context(cwd=checkout_root): |
| # Format test data with start directory. |
| start_dir = api.path["start_dir"] |
| for petal in PETALS_TEST_DATA_TEMPLATE: |
| petal["path"] = petal["path"].format(start_dir=start_dir) |
| |
| # Grab petal branch info. |
| petals = api.jiri.project( |
| list_remote=True, |
| test_data=PETALS_TEST_DATA_TEMPLATE, |
| ).json.output |
| |
| # Maintain list of snapped petal local git directories. |
| petal_paths = [] |
| |
| # Snap any petal branches which exist. |
| for petal in petals: |
| if petal["name"] in skip_snap_projects or petal["name"] == integration_project: |
| continue |
| petal_remote = api.sso.sso_to_https(petal["remote"]) |
| petal_branch_revision = api.git.get_remote_branch_head( |
| step_name="get %s branch HEAD" % petal["name"], |
| url=petal_remote, |
| branch=target_branch, |
| ) |
| if petal_branch_revision: |
| petal_path = api.path.abs_to_path(petal["path"]) |
| snap_message = SNAP_MESSAGE.format( |
| prefix=api.release._COMMIT_MESSAGE_PREFIX, |
| revision=api.release.shorten_revision(petal["revision"]), |
| ) |
| # Snap to target petal revision. |
| api.git.snap_branch( |
| step_name="snap %s" % petal["name"], |
| snap_ref=petal["revision"], |
| branch=target_branch, |
| message=snap_message, |
| path=petal_path, |
| push=False, |
| ) |
| |
| with api.context(cwd=petal_path): |
| new_petal_revision = api.git.get_hash() |
| # Skip if the snap revision, target petal revision, and remote |
| # revision are all equal. This means that the petal branch is |
| # already consistent with integration. |
| if new_petal_revision == petal["revision"] == petal_branch_revision: |
| continue |
| # For snapped petal, sync its manifest revision to the new revision |
| # created by the snap. |
| with api.context(cwd=checkout_root): |
| api.jiri.edit_manifest( |
| manifest=petal["manifest"], |
| projects=[(petal["name"], new_petal_revision)], |
| ) |
| petal_paths.append(petal_path) |
| return petal_paths |
| |
| |
| def GenTests(api): |
| default_revision = "foo" |
| default_project = "integration" |
| default_manifest = "integration" |
| default_remote = "sso://fuchsia/integration" |
| default_branch = "releases/targetbranch" |
| empty_output = api.raw_io.stream_output_text("") |
| branch_does_not_exist = api.step_data( |
| "check inputs.get target branch HEAD", |
| empty_output, |
| ) |
| revision_has_no_release_history = api.release.ref_to_release_version( |
| "", |
| nesting="resolve release version", |
| retcode=1, |
| ) |
| version_does_not_exist = api.step_data( |
| "resolve release version.get remote tag", |
| empty_output, |
| ) |
| source_version = api.release.ref_to_release_version( |
| "releases/0.20200101.0.1", |
| nesting="resolve release version.check rollback inputs", |
| ) |
| rollback_branch_version = api.release.ref_to_release_version( |
| "releases/0.20200101.0.2", |
| nesting="resolve release version.check rollback inputs", |
| rep=2, |
| ) |
| invalid_rollback_branch_version = api.release.ref_to_release_version( |
| "releases/0.20200101.1.1", |
| nesting="resolve release version.check rollback inputs", |
| rep=2, |
| ) |
| rollback_target_version = api.release.ref_to_release_version( |
| "releases/0.20200101.0.3", |
| nesting="resolve release version.check rollback inputs", |
| rep=3, |
| ) |
| matching_source_version = api.release.ref_to_release_version( |
| "releases/0.20200101.0.1", |
| nesting="snap integration", |
| ) |
| no_matching_source_version = api.release.ref_to_release_version( |
| "", |
| nesting="snap integration", |
| retcode=1, |
| ) |
| |
| yield ( |
| api.status_check.test("mutually_exclusive_rev_input", status="failure") |
| + api.properties( |
| revision=default_revision, |
| lkg_builders=["project/bucket/builder"], |
| ) |
| ) |
| yield (api.status_check.test("missing_rev_input", status="failure")) |
| |
| yield ( |
| api.status_check.test("invalid_source_branch_input", status="failure") |
| + api.properties(source_branch="invalidbranch") |
| ) |
| |
| yield ( |
| api.status_check.test("invalid_target_branch_input", status="failure") |
| + api.properties(revision=default_revision, target_branch="invalidbranch") |
| ) |
| |
| yield ( |
| api.status_check.test("branch_does_not_exist", status="failure") |
| + api.properties(revision=default_revision, target_branch=default_branch) |
| + branch_does_not_exist |
| ) |
| |
| yield ( |
| api.status_check.test("release_version_conflict", status="failure") |
| + api.properties( |
| revision=default_revision, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| target_branch=default_branch, |
| ) |
| + revision_has_no_release_history |
| ) |
| |
| yield ( |
| api.status_check.test("revision_from_main") |
| + api.properties( |
| revision=default_revision, |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| downstream_builders=["a/b/c"], |
| ) |
| + revision_has_no_release_history |
| + version_does_not_exist |
| + no_matching_source_version |
| + api.step_data( |
| "snap petals.get bar branch HEAD", |
| api.raw_io.stream_output_text("deadbeef\trefs/heads/branch"), |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("rollback") |
| + api.properties( |
| revision=default_revision, |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| rollback=True, |
| ) |
| + source_version |
| + rollback_branch_version |
| + rollback_target_version |
| + version_does_not_exist |
| + no_matching_source_version |
| ) |
| |
| yield ( |
| api.status_check.test("invalid_rollback", status="failure") |
| + api.properties( |
| revision=default_revision, |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| rollback=True, |
| ) |
| + source_version |
| + invalid_rollback_branch_version |
| ) |
| |
| yield ( |
| api.status_check.test("version_to_version") |
| + api.properties( |
| revision=default_revision, |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| ) |
| + revision_has_no_release_history |
| + version_does_not_exist |
| + matching_source_version |
| ) |
| |
| yield ( |
| api.status_check.test("lkgr_from_main") |
| + api.properties( |
| lkg_builders=["project/bucket/builder"], |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| skip_snap_projects=["foo"], |
| ) |
| + revision_has_no_release_history |
| + version_does_not_exist |
| + no_matching_source_version |
| ) |
| |
| yield ( |
| api.status_check.test("mismatched_milestones") |
| + api.properties( |
| lkg_builders=["project/bucket/builder"], |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| must_match_milestone=True, |
| ) |
| + revision_has_no_release_history |
| + version_does_not_exist |
| + api.step_data( |
| "resolve release version.check milestone match.get major number", |
| api.file.read_text("1\n"), |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("noop_snap", status="failure") |
| + api.properties( |
| source_branch="releases/sourcebranch", |
| target_branch=default_branch, |
| ) |
| ) |
