| # Copyright 2020 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Snap an integration.git release branch. |
| |
| Branching Modes |
| - Revision: Snap the branch to a specific revision. |
| - LKGR: Snap the branch using the LKGR of a set of builders. |
| - Branch: Snap the branch to another branch. |
| |
| Options |
| - Experimental: Snap the branch using the "add track" feature. |
| See release/api.py for more details. |
| - Rollback: Enable rolling back to a previous candidate version. |
| - Dryrun: Run recipe without modifying anything remotely. |
| """ |
| |
| from urllib.parse import urlparse |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2 |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2 |
| from PB.recipe_engine.result import RawResult |
| from PB.recipes.fuchsia.release.snap_branch import InputProperties |
| |
| DEPS = [ |
| "fuchsia/checkout", |
| "fuchsia/git", |
| "fuchsia/git_checkout", |
| "fuchsia/jiri", |
| "fuchsia/lkg", |
| "fuchsia/release", |
| "fuchsia/sso", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| "recipe_engine/time", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| SNAP_MESSAGE = """\ |
| {prefix} Snap to {revision}{maybe_equal_revision} |
| |
| Release-Operation-Build: {build_url} |
| """ |
| PROJECTS_TEST_DATA_TEMPLATE = [ |
| { |
| "name": "foo", |
| "path": "{start_dir}/local/path/to/foo", |
| "remote": "https://fuchsia.googlesource.com/foo", |
| "revision": "snapme", |
| "manifest": "local/path/to/foo/manifest", |
| }, |
| { |
| "name": "bar", |
| "path": "{start_dir}/local/path/to/bar", |
| "remote": "https://fuchsia.googlesource.com/bar", |
| # Simulate revision that does not need snapping. |
| "revision": "deadbeef", |
| "manifest": "local/path/to/bar/manifest", |
| }, |
| { |
| # This should always be skipped. |
| "name": "integration", |
| "path": "{start_dir}/local/path/to/integration", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "revision": "skipme", |
| "manifest": "local/path/to/integration/manifest", |
| }, |
| ] |
| |
| |
| def RunSteps(api, props): |
| with api.step.nest("check inputs"): |
| if not ( |
| bool(props.revision) ^ bool(props.lkg_builders) ^ bool(props.source_branch) |
| ): |
| return RawResult( |
| summary_markdown="must specify exactly one of `revision`, " |
| "`lkg_builders`, or `source_branch`", |
| status=common_pb2.FAILURE, |
| ) |
| if props.source_branch and not api.release.validate_branch(props.source_branch): |
| return RawResult( |
| summary_markdown='`source_branch` must start with "releases/"', |
| status=common_pb2.FAILURE, |
| ) |
| if not api.release.validate_branch(props.target_branch): |
| return RawResult( |
| summary_markdown='`target_branch` must start with "releases/"', |
| status=common_pb2.FAILURE, |
| ) |
| # Assert target branch exists remotely. |
| https_remote = api.sso.sso_to_https(props.remote) |
| branch_revision = api.git.get_remote_branch_head( |
| step_name="get target branch HEAD", |
| url=https_remote, |
| branch=props.target_branch, |
| ) |
| if not branch_revision: |
| return RawResult( |
| summary_markdown=( |
| f"target branch {props.target_branch} does not exist" |
| ), |
| status=common_pb2.FAILURE, |
| ) |
| |
| # Resolve revision from one of `revision`, `lkg_builders`, or `source_branch`. |
| with api.step.nest("resolve source revision"): |
| revision = props.revision |
| if props.lkg_builders: |
| revision = api.lkg.revision( |
| step_name="get last-known-good revision", |
| builders=props.lkg_builders, |
| test_data="lkgrevision", |
| ) |
| elif props.source_branch: |
| with api.context(infra_steps=True): |
| revision = api.git.get_remote_branch_head( |
| step_name="get source branch HEAD", |
| url=https_remote, |
| branch=props.source_branch, |
| ) |
| if revision == branch_revision: |
| return RawResult( |
| summary_markdown=( |
| f"resolved snap revision {revision} is already current " |
| f"revision on {props.target_branch}" |
| ), |
| status=common_pb2.FAILURE, |
| ) |
| |
| checkout = api.checkout.fuchsia_with_options( |
| build_input=build_pb2.Build.Input( |
| gitiles_commit=common_pb2.GitilesCommit( |
| id=revision, |
| project=props.project, |
| host=urlparse(https_remote).hostname, |
| ), |
| ), |
| project=props.project, |
| manifest=props.manifest, |
| remote=props.remote, |
| # Skip for faster checkout. |
| fetch_packages=False, |
| ) |
| integration_root = checkout.project_path(props.project) |
| |
| with api.context(cwd=integration_root, infra_steps=True): |
| # Ensure tags are up-to-date. |
| api.git.fetch("origin", tags=True, refspec=props.target_branch) |
| |
| with api.step.nest("resolve release version"): |
| candidate_version_increment = props.candidate_version_increment |
| if props.rollback: |
| with api.step.nest("check rollback inputs"): |
| with api.context(infra_steps=True): |
| rollback_version = api.release.ref_to_release_version( |
| ref=revision, repo_path=integration_root |
| ) |
| branch_version = api.release.ref_to_release_version( |
| ref=f"origin/{props.target_branch}", |
| repo_path=integration_root, |
| ) |
| # Ensure the target revision is a previous candidate version of |
| # target branch's HEAD. |
| if ( |
| rollback_version.major != branch_version.major |
| or rollback_version.date != branch_version.date |
| or rollback_version.release != branch_version.release |
| or rollback_version.candidate >= branch_version.candidate |
| ): |
| return RawResult( |
| summary_markdown="cannot rollback to non-previous " |
| "candidate version: rollback version %s is not a " |
| "previous candidate version of %s's HEAD, %s" |
| % ( |
| rollback_version.tag_name, |
| props.target_branch, |
| branch_version.tag_name, |
| ), |
| status=common_pb2.FAILURE, |
| ) |
| # Resolve the new release version, which is the next candidate |
| # version on the branch. |
| release_version = api.release.get_next_candidate_version( |
| ref=f"origin/{props.target_branch}", |
| add_track=props.add_track, |
| candidate_version_increment=candidate_version_increment, |
| repo_path=integration_root, |
| ) |
| else: |
| # If there are existing candidates on the target revision, then get |
| # the next candidate version. |
| release_version = api.release.get_next_candidate_version( |
| ref=revision, |
| add_track=props.add_track, |
| candidate_version_increment=candidate_version_increment, |
| repo_path=integration_root, |
| ) |
| # Otherwise, get the next release version. This means that the |
| # target revision has no release history, and is almost certainly |
| # from the main branch. |
| if not release_version: |
| # Look at origin, since target branch will not have a local copy |
| # at this point. |
| release_version = api.release.get_next_release_version( |
| ref=f"origin/{props.target_branch}", |
| date=api.time.utcnow().date().strftime("%Y%m%d"), |
| repo_path=integration_root, |
| candidate_version_increment=candidate_version_increment, |
| release_version_initialization=props.release_version_initialization, |
| ) |
| if api.git.get_remote_tag(url=https_remote, tag=release_version.tag_name): |
| return RawResult( |
| summary_markdown=( |
| f"release version conflict: attempting snap would conflict " |
| f"with existing tag {release_version.tag_name}, indicating " |
| f"source revision {revision} has already moved forward" |
| ), |
| status=common_pb2.FAILURE, |
| ) |
| if props.must_match_milestone: |
| head_checkout, _ = api.git_checkout( |
| https_remote, |
| revision=revision, |
| cache=False, |
| ) |
| # If the resolved release version's milestone number is not equal |
| # to the milestone number at HEAD, do nothing. |
| with api.step.nest("check milestone match"): |
| if release_version.major != api.release.get_major_number(head_checkout): |
| return RawResult( |
| summary_markdown="skipping snap: milestone numbers do " |
| "not match", |
| status=common_pb2.SUCCESS, |
| ) |
| |
| with api.step.nest("snap integration"), api.context(infra_steps=True): |
| # If the source revision has an associated release version, add it |
| # to the commit message. |
| source_version = api.release.ref_to_release_version( |
| ref=revision, |
| repo_path=integration_root, |
| ) |
| maybe_equal_revision = "" |
| if source_version: |
| maybe_equal_revision = f" (equal to {source_version})" |
| snap_message = SNAP_MESSAGE.format( |
| prefix=api.release._COMMIT_MESSAGE_PREFIX, |
| revision=api.release.shorten_revision(revision), |
| maybe_equal_revision=maybe_equal_revision, |
| build_url=api.buildbucket.build_url(), |
| ) |
| # Snap to target revision. |
| api.git.snap_branch( |
| snap_ref=revision, |
| branch=props.target_branch, |
| message=snap_message, |
| path=integration_root, |
| push=False, |
| ) |
| |
| snapped_project_paths = [] |
| if props.snap_projects: |
| with api.step.nest("snap projects"), api.context(infra_steps=True): |
| snapped_project_paths = snap_projects( |
| api=api, |
| target_branch=props.target_branch, |
| checkout_root=checkout.root_dir, |
| integration_project=props.project, |
| ) |
| for path in snapped_project_paths: |
| with api.context(cwd=path): |
| api.git.push( |
| step_name="push commit", |
| refs=f"HEAD:refs/heads/{props.target_branch}", |
| dryrun=props.dryrun, |
| ) |
| |
| with api.context(cwd=integration_root, infra_steps=True), api.step.nest( |
| "push integration" |
| ) as presentation: |
| # Squash integration snap and project snaps together. |
| if snapped_project_paths: |
| api.git.add(step_name="add project snaps", add_all=True) |
| api.git.commit( |
| step_name="squash project snaps", |
| amend=True, |
| no_edit=True, |
| ) |
| |
| release_revision = api.git.get_hash() |
| # Tag the snap commit. |
| api.git.tag(release_version.tag_name, step_name="tag release") |
| # Push commit and release version. |
| api.git.push( |
| step_name="push release", |
| refs=[ |
| f"HEAD:refs/heads/{props.target_branch}", |
| f"refs/tags/{release_version.tag_name}", |
| ], |
| atomic=True, |
| dryrun=props.dryrun, |
| ) |
| api.release.set_output_properties( |
| presentation, |
| release_revision, |
| release_version, |
| https_remote, |
| ) |
| |
| if not props.dryrun and props.downstream_builders: |
| api.release.lookup_builds( |
| builders=props.downstream_builders, |
| revision=release_revision, |
| remote=https_remote, |
| ) |
| |
| |
| def snap_projects(api, target_branch, checkout_root, integration_project): |
| """Snap any existing project branches to their global integration revisions. |
| Sync manifest revisions in global integration to their new snap revisions |
| created by the snaps. |
| |
| Args: |
| target_branch (str): Branch to snap to. |
| checkout_root (Path): Path including all project checkouts. |
| |
| Returns: |
| list(Path): Paths of snapped project directories. |
| """ |
| with api.context(cwd=checkout_root): |
| # Format test data with start directory. |
| start_dir = api.path.start_dir |
| for project in PROJECTS_TEST_DATA_TEMPLATE: |
| project["path"] = project["path"].format(start_dir=start_dir) |
| projects = api.jiri.project( |
| list_remote=True, |
| test_data=PROJECTS_TEST_DATA_TEMPLATE, |
| ).json.output |
| |
| project_paths = [] |
| for project in projects: |
| if project["name"] == integration_project: |
| continue |
| project_remote = api.sso.sso_to_https(project["remote"]) |
| project_branch_revision = api.git.get_remote_branch_head( |
| step_name=f"get {project['name']} branch HEAD", |
| url=project_remote, |
| branch=target_branch, |
| ) |
| if project_branch_revision: |
| project_path = api.path.abs_to_path(project["path"]) |
| snap_message = SNAP_MESSAGE.format( |
| prefix=api.release._COMMIT_MESSAGE_PREFIX, |
| revision=api.release.shorten_revision(project["revision"]), |
| maybe_equal_revision="", |
| build_url=api.buildbucket.build_url(), |
| ) |
| # Snap to target project revision. |
| api.git.snap_branch( |
| step_name=f"snap {project['name']}", |
| snap_ref=project["revision"], |
| branch=target_branch, |
| message=snap_message, |
| path=project_path, |
| push=False, |
| ) |
| |
| with api.context(cwd=project_path): |
| new_project_revision = api.git.get_hash() |
| # Skip if the snap revision, target project revision, and remote |
| # revision are all equal. This means that the project branch is |
| # already consistent with integration. |
| if ( |
| new_project_revision |
| == project["revision"] |
| == project_branch_revision |
| ): |
| continue |
| # For snapped project, sync its manifest revision to the new revision |
| # created by the snap. |
| with api.context(cwd=checkout_root): |
| api.jiri.edit_manifest( |
| manifest=project["manifest"], |
| projects=[(project["name"], new_project_revision)], |
| ) |
| project_paths.append(project_path) |
| return project_paths |
| |
| |
| def GenTests(api): |
| default_revision = "foo" |
| default_project = "integration" |
| default_manifest = "integration" |
| default_remote = "sso://fuchsia/integration" |
| default_branch = "releases/targetbranch" |
| empty_output = api.raw_io.stream_output_text("") |
| branch_does_not_exist = api.step_data( |
| "check inputs.get target branch HEAD", |
| empty_output, |
| ) |
| revision_has_no_release_history = api.release.ref_to_release_version( |
| "", |
| nesting="resolve release version", |
| retcode=1, |
| ) |
| version_does_not_exist = api.step_data( |
| "resolve release version.get remote tag", |
| empty_output, |
| ) |
| source_version = api.release.ref_to_release_version( |
| "releases/0.20200101.0.1", |
| nesting="resolve release version.check rollback inputs", |
| ) |
| rollback_branch_version = api.release.ref_to_release_version( |
| "releases/0.20200101.0.2", |
| nesting="resolve release version.check rollback inputs", |
| rep=2, |
| ) |
| invalid_rollback_branch_version = api.release.ref_to_release_version( |
| "releases/0.20200101.1.1", |
| nesting="resolve release version.check rollback inputs", |
| rep=2, |
| ) |
| rollback_target_version = api.release.ref_to_release_version( |
| "releases/0.20200101.0.3", |
| nesting="resolve release version.check rollback inputs", |
| rep=3, |
| ) |
| matching_source_version = api.release.ref_to_release_version( |
| "releases/0.20200101.0.1", |
| nesting="snap integration", |
| ) |
| no_matching_source_version = api.release.ref_to_release_version( |
| "", |
| nesting="snap integration", |
| retcode=1, |
| ) |
| |
| yield ( |
| api.test("mutually_exclusive_rev_input", status="FAILURE") |
| + api.properties( |
| revision=default_revision, |
| lkg_builders=["project/bucket/builder"], |
| ) |
| ) |
| yield api.test("missing_rev_input", status="FAILURE") |
| |
| yield ( |
| api.test("invalid_source_branch_input", status="FAILURE") |
| + api.properties(source_branch="invalidbranch") |
| ) |
| |
| yield ( |
| api.test("invalid_target_branch_input", status="FAILURE") |
| + api.properties(revision=default_revision, target_branch="invalidbranch") |
| ) |
| |
| yield ( |
| api.test("branch_does_not_exist", status="FAILURE") |
| + api.properties(revision=default_revision, target_branch=default_branch) |
| + branch_does_not_exist |
| ) |
| |
| # TODO(atyfto): Reduce duplication of properties across test cases. |
| yield ( |
| api.test("release_version_conflict", status="FAILURE") |
| + api.properties( |
| revision=default_revision, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| target_branch=default_branch, |
| ) |
| + revision_has_no_release_history |
| ) |
| |
| yield ( |
| api.test("revision_from_main") |
| + api.properties( |
| revision=default_revision, |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| downstream_builders=["a/b/c"], |
| snap_projects=True, |
| ) |
| + revision_has_no_release_history |
| + version_does_not_exist |
| + no_matching_source_version |
| + api.step_data( |
| "snap projects.get bar branch HEAD", |
| api.raw_io.stream_output_text("deadbeef\trefs/heads/branch"), |
| ) |
| ) |
| |
| yield ( |
| api.test("rollback") |
| + api.properties( |
| revision=default_revision, |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| rollback=True, |
| snap_projects=True, |
| ) |
| + source_version |
| + rollback_branch_version |
| + rollback_target_version |
| + version_does_not_exist |
| + no_matching_source_version |
| ) |
| |
| yield ( |
| api.test("invalid_rollback", status="FAILURE") |
| + api.properties( |
| revision=default_revision, |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| rollback=True, |
| snap_projects=True, |
| ) |
| + source_version |
| + invalid_rollback_branch_version |
| ) |
| |
| yield ( |
| api.test("version_to_version") |
| + api.properties( |
| revision=default_revision, |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| snap_projects=True, |
| ) |
| + revision_has_no_release_history |
| + version_does_not_exist |
| + matching_source_version |
| ) |
| yield ( |
| api.test("mismatched_milestones") |
| + api.properties( |
| lkg_builders=["project/bucket/builder"], |
| target_branch=default_branch, |
| project=default_project, |
| manifest=default_manifest, |
| remote=default_remote, |
| must_match_milestone=True, |
| snap_projects=True, |
| ) |
| + revision_has_no_release_history |
| + version_does_not_exist |
| + api.step_data( |
| "resolve release version.check milestone match.get major number", |
| api.file.read_text("1\n"), |
| ) |
| ) |
| |
| yield ( |
| api.test("noop_snap", status="FAILURE") |
| + api.properties( |
| source_branch="releases/sourcebranch", |
| target_branch=default_branch, |
| ) |
| ) |