| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for updating a third party repo to the greatest released version. |
| |
| Searches the `import_from` repo for tags of the form "v${SEMANTIC_VERSION}", where |
| SEMANTIC_VERSION looks like ${MAJOR}.${MINOR}.${PATCH}. See https://semver.org/. |
| |
| Then updates the jiri manifest `import_in` in repo `project` with the revision of |
| the greatest semantic version. |
| |
| Then updates the main branch of `import_from` to that same revision. |
| |
| The dry_run property allows testing the auto-roller end-to-end. This is |
| useful because now we can have an auto-roller in staging, and we can block |
| updates behind 'dry_run' as a sort of feature gate. It is passed to |
| api.auto_roller.attempt_roll() which handles committing changes. |
| """ |
| |
| from PB.recipes.fuchsia.third_party_repo_roller import InputProperties |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/gerrit", |
| "fuchsia/git", |
| "fuchsia/gitiles", |
| "fuchsia/jiri", |
| "fuchsia/sso", |
| "fuchsia/status_check", |
| "recipe_engine/context", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/python", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| COMMIT_MESSAGE = """[roll] Roll {header} |
| {body} |
| """ |
| |
| COMMIT_HEADER = """{project} {old}..{new} ({count} commits)""" |
| |
| # These footers must be set for the roll_comment recipe to be able to comment |
| # on the rolled CLs. |
| COMMENT_FOOTERS = """Rolled-Repo: {repo} |
| Rolled-Commits: {old}..{new}""" |
| |
| # If we're embedding the original commit message, prepend 'Original-' to lines |
| # which begin with these tags. |
| ESCAPE_TAGS = [ |
| "Reviewed-on:", |
| "Bug:", |
| "Fixed:", |
| ] |
| # If we're embedding the original commit message, remove lines which contain |
| # these tags. |
| FILTER_TAGS = [ |
| "Change-Id:", |
| "Reviewed-by:", |
| "Signed-off-by:", |
| "Acked-by:", |
| "CC:", |
| "Tested-by:", |
| "Commit-Queue:", |
| "Testability-Review:", |
| "API-Review:", |
| "Auto-Submit:", |
| "Fuchsia-Auto-Submit:", |
| ] |
| |
| TEST_DATA_OLD_REV = "123abc" |
| |
| |
| def RunSteps(api, props): |
| with api.context(infra_steps=True): |
| if props.owners: |
| owners_step = api.step("owners", None) |
| owners_step.presentation.step_summary_text = ", ".join(props.owners) |
| |
| with api.step.nest("checkout"): |
| api.jiri.init() |
| api.jiri.import_manifest(props.manifest, props.remote, name=props.project) |
| api.jiri.update(run_hooks=False, fetch_packages=False) |
| |
| manifest_project, import_from_project = api.jiri.project( |
| projects=(props.project, props.import_from), |
| test_data=( |
| { |
| "name": props.project, |
| "path": str(api.path["start_dir"].join(props.project)), |
| "remote": props.remote, |
| }, |
| { |
| "name": props.import_from, |
| "path": str(api.path["start_dir"].join(props.import_from)), |
| "remote": "https://fuchsia.googlesource.com/%s" % props.import_from, |
| "revision": TEST_DATA_OLD_REV, |
| }, |
| ), |
| ).json.output |
| |
| project_dir = api.path.abs_to_path(manifest_project["path"]) |
| with api.step.nest("edit manifest"), api.context(cwd=project_dir): |
| roll_from_repo = import_from_project["remote"] |
| roll_from_repo = api.sso.sso_to_https(roll_from_repo) |
| |
| old_rev = import_from_project["revision"] |
| new_rev = resolve_new_revision(api, roll_from_repo) |
| |
| if old_rev == new_rev: |
| api.python.succeeding_step("nothing to roll", "manifest up-to-date") |
| return api.auto_roller.nothing_to_roll() |
| |
| log = api.gitiles.log( |
| roll_from_repo, "%s..%s" % (old_rev, new_rev), step_name="log" |
| ) |
| if not log: |
| api.python.infra_failing_step( |
| "detected backwards roll", |
| "expected %s to precede %s in git history" |
| % (old_rev[:7], new_rev[:7]), |
| ) |
| |
| api.jiri.edit_manifest( |
| props.import_in, projects=[(props.import_from, new_rev)] |
| ) |
| |
| commit_message = write_commit_message(log, props.import_from, old_rev, new_rev) |
| |
| # Land the changes. |
| change = api.auto_roller.attempt_roll( |
| api.gerrit.host_from_remote_url(props.remote), |
| gerrit_project=props.project, |
| repo_dir=project_dir, |
| commit_message=commit_message, |
| dry_run=props.dry_run, |
| raise_on_failure=False, |
| ) |
| |
| roll_result = api.auto_roller.raw_result(change) |
| |
| # For clarity we update the main branch the same revision |
| # that is pinned in the jiri manifest. |
| import_from_dir = api.path.abs_to_path(import_from_project["path"]) |
| with api.context(cwd=import_from_dir): |
| api.git.push( |
| "%s:main" % new_rev, |
| step_name="fast forward main", |
| dryrun=props.dry_run, |
| ) |
| |
| return roll_result |
| |
| |
| def write_commit_message( |
| log, |
| import_from, |
| old_rev, |
| new_rev, |
| ): |
| commit_lines = [] |
| for commit in log: |
| commit_label = commit["id"][:7] |
| commit_lines.append( |
| "{commit} {subject}".format( |
| commit=commit_label, |
| subject=commit["message"].splitlines()[0], |
| ) |
| ) |
| |
| header = COMMIT_HEADER.format( |
| project=import_from, |
| old=old_rev[:7], |
| new=new_rev[:7], |
| count=len(log), |
| ) |
| body = "\n" + "\n".join(commit_lines) |
| |
| return COMMIT_MESSAGE.format(header=header, body=body) |
| |
| |
| def to_semantic_version_tuple(git_ref): |
| """Returns a tuple of ints: (major, minor, patch) or None.""" |
| ref_prefix = "refs/tags/v" |
| if not git_ref.startswith(ref_prefix): |
| return None |
| sem_ver_str = git_ref[len(ref_prefix) :] |
| sem_ver_parts = sem_ver_str.split(".") |
| if len(sem_ver_parts) != 3: |
| return None |
| try: |
| return tuple(int(x) for x in sem_ver_parts) |
| except ValueError: |
| return None |
| |
| |
| def resolve_new_revision(api, repo): |
| with api.step.nest("resolve new revision") as presentation: |
| ls_output = api.git( |
| "ls-remote", |
| "ls-remote", |
| repo, |
| "--tags", |
| "refs/tags/v*", |
| stdout=api.raw_io.output(), |
| ).stdout |
| # Each line looks like: |
| # <commit ID> <refs/tags/...> |
| refs_commits = [ |
| tuple(reversed(commit_ref.split())) |
| for commit_ref in ls_output.split("\n") |
| if commit_ref |
| ] |
| semantic_versions_commits = sorted( |
| (to_semantic_version_tuple(ref), commit) |
| for ref, commit in refs_commits |
| if to_semantic_version_tuple(ref) |
| ) |
| assert semantic_versions_commits, "failed to resolve commit to roll" |
| greatest_version, greatest_commit = semantic_versions_commits[-1] |
| presentation.logs["greatest semantic version"] = "v%d.%d.%d" % greatest_version |
| return greatest_commit |
| |
| |
| def GenTests(api): |
| DEFAULT_NEW_REV = "fc4dc762688d2263b254208f444f5c0a4b91bc07" |
| |
| def properties(**kwargs): |
| props = dict( |
| project="integration", |
| manifest="minimal", |
| remote="https://fuchsia.googlesource.com/integration", |
| import_in="stem", |
| import_from="fuchsia", |
| owners=["nobody@google.com", "noreply@google.com"], |
| ) |
| props.update(**kwargs) |
| return api.properties(**props) |
| |
| def ls_remote(greatest_release_revision="123abd"): |
| return api.step_data( |
| "edit manifest.resolve new revision.ls-remote", |
| api.raw_io.stream_output( |
| "\n".join( |
| [ |
| "a21bf2e6466095c7a2cdb991017da9639cf496e5 refs/tags/v2.6.0", |
| "%s refs/tags/v2.10.1" % greatest_release_revision, |
| "419c94b6e07f93610e5124a0be0a7ee7140006ed refs/tags/v2.6.1rc1", |
| "e8ae137c96444ea313485ed1118c5e43b2099cf1 foo/bar/baz", |
| "e8ae137c96444ea313485ed1118c5e43b2099cf2 refs/tags/v1.2.3.4", |
| ] |
| ) |
| ), |
| ) |
| |
| yield ( |
| api.status_check.test("successful") |
| + properties() |
| + ls_remote() |
| + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # If the repo is already at the greatest release, the recipe should not attempt |
| # a roll. |
| yield (api.status_check.test("noop") + properties() + ls_remote(TEST_DATA_OLD_REV)) |
| |
| # If the currently pinned revision and the main revision are not equal, |
| # but the gitiles log between them is empty, that probably means that the |
| # currently pinned revision is actually *ahead* of main, according to |
| # `gitiles log`. This sometimes happens, possibly due to GoB |
| # synchronization issues, and we should fail instead of rolling backwards. |
| yield ( |
| api.status_check.test("backwards_roll", status="infra_failure") |
| + properties() |
| + ls_remote() |
| + api.gitiles.log("edit manifest.log", "A", n=0) |
| ) |