| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for updating a third party repo to the greatest released version. |
| |
| Searches the `import_from` repo for tags of the form "v${SEMANTIC_VERSION}", |
| where SEMANTIC_VERSION looks like ${MAJOR}.${MINOR}.${PATCH}. See |
| https://semver.org/. |
| |
| Then updates the jiri manifest `import_in` in repo `project` with the revision |
| of the greatest semantic version. |
| |
| Then updates the main branch of `import_from` to that same revision. |
| |
| The dry_run property allows testing the auto-roller end-to-end. This is useful |
| because now we can have an auto-roller in staging, and we can block updates |
| behind 'dry_run' as a sort of feature gate. It is passed to |
| api.auto_roller.attempt_roll() which handles committing changes. |
| """ |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2 |
| from PB.recipes.fuchsia.third_party_repo_roller import InputProperties |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/checkout", |
| "fuchsia/git", |
| "fuchsia/gitiles", |
| "fuchsia/jiri", |
| "fuchsia/sso", |
| "recipe_engine/context", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| COMMIT_MESSAGE = """[roll] Roll {header} |
| {body} |
| """ |
| |
| COMMIT_HEADER = """{project} {old}..{new} ({count} commits)""" |
| |
| # These footers must be set for the roll_comment recipe to be able to comment |
| # on the rolled CLs. |
| COMMENT_FOOTERS = """Rolled-Repo: {repo} |
| Rolled-Commits: {old}..{new}""" |
| |
| # If we're embedding the original commit message, prepend 'Original-' to lines |
| # which begin with these tags. |
| ESCAPE_TAGS = [ |
| "Reviewed-on:", |
| "Bug:", |
| "Fixed:", |
| ] |
| # If we're embedding the original commit message, remove lines which contain |
| # these tags. |
| FILTER_TAGS = [ |
| "Change-Id:", |
| "Reviewed-by:", |
| "Signed-off-by:", |
| "Acked-by:", |
| "CC:", |
| "Tested-by:", |
| "Commit-Queue:", |
| "Testability-Review:", |
| "API-Review:", |
| "Auto-Submit:", |
| "Fuchsia-Auto-Submit:", |
| ] |
| |
| |
| def RunSteps(api, props): |
| if props.owners: |
| api.step.empty("owners", step_text=", ".join(props.owners)) |
| |
| checkout = api.checkout.fuchsia_with_options( |
| manifest=props.manifest, |
| remote=props.remote, |
| project=props.project, |
| # Ignore the build input; we should always check out the manifest |
| # repository at HEAD before updating the manifest to reduce the |
| # likelihood of merge conflicts. |
| build_input=build_pb2.Build.Input(), |
| use_lock_file=True, |
| ) |
| manifest_project = checkout.project(props.project) |
| import_from_project = checkout.project(props.import_from) |
| project_path = checkout.project_path(props.project) |
| |
| with api.step.nest("edit manifest"), api.context( |
| cwd=project_path, infra_steps=True |
| ): |
| roll_from_repo = import_from_project["remote"] |
| roll_from_repo = api.sso.sso_to_https(roll_from_repo) |
| |
| old_rev = import_from_project["revision"] |
| new_rev = resolve_new_revision(api, roll_from_repo) |
| |
| if old_rev == new_rev: |
| api.step.empty("nothing to roll", step_text="manifest up-to-date") |
| return api.auto_roller.nothing_to_roll() |
| |
| log = api.gitiles.log(roll_from_repo, f"{old_rev}..{new_rev}", step_name="log") |
| if not log: |
| api.step.empty( |
| "detected backwards roll", |
| step_text=f"expected {old_rev[:7]} to precede {new_rev[:7]} in git history", |
| status=api.step.INFRA_FAILURE, |
| ) |
| |
| api.jiri.edit_manifest(props.import_in, projects=[(props.import_from, new_rev)]) |
| |
| commit_message = write_commit_message(log, props.import_from, old_rev, new_rev) |
| |
| # Land the changes. |
| change = api.auto_roller.attempt_roll( |
| api.auto_roller.Options( |
| remote=props.remote, |
| dry_run=props.dry_run, |
| roller_owners=props.owners, |
| ), |
| repo_dir=project_path, |
| commit_message=commit_message, |
| raise_on_failure=False, |
| ) |
| |
| roll_result = api.auto_roller.raw_result(change) |
| |
| # For clarity we update the main branch the same revision |
| # that is pinned in the jiri manifest. |
| import_from_dir = api.path.abs_to_path(import_from_project["path"]) |
| with api.context(cwd=import_from_dir): |
| api.git.push( |
| f"{new_rev}:main", |
| step_name="fast forward main", |
| dryrun=props.dry_run, |
| ) |
| |
| return roll_result |
| |
| |
| def write_commit_message( |
| log, |
| import_from, |
| old_rev, |
| new_rev, |
| ): |
| commit_lines = [] |
| for commit in log: |
| commit_label = commit["id"][:7] |
| commit_lines.append(f"{commit_label} {commit['message'].splitlines()[0]}") |
| |
| header = COMMIT_HEADER.format( |
| project=import_from, |
| old=old_rev[:7], |
| new=new_rev[:7], |
| count=len(log), |
| ) |
| body = "\n" + "\n".join(commit_lines) |
| |
| return COMMIT_MESSAGE.format(header=header, body=body) |
| |
| |
| def to_semantic_version_tuple(git_ref): |
| """Returns a tuple of ints: (major, minor, patch) or None.""" |
| ref_prefix = "refs/tags/v" |
| if not git_ref.startswith(ref_prefix): |
| return None |
| sem_ver_parts = git_ref.removeprefix(ref_prefix).split(".") |
| if len(sem_ver_parts) != 3: |
| return None |
| try: |
| return tuple(int(x) for x in sem_ver_parts) |
| except ValueError: |
| return None |
| |
| |
| def resolve_new_revision(api, repo): |
| with api.step.nest("resolve new revision") as presentation: |
| ls_output = api.git( |
| "ls-remote", |
| "ls-remote", |
| repo, |
| "--tags", |
| "refs/tags/v*", |
| stdout=api.raw_io.output_text(), |
| ).stdout |
| # Each line looks like: |
| # <commit ID> <refs/tags/...> |
| refs_commits = [ |
| tuple(reversed(commit_ref.split())) |
| for commit_ref in ls_output.split("\n") |
| if commit_ref |
| ] |
| semantic_versions_commits = sorted( |
| (to_semantic_version_tuple(ref), commit) |
| for ref, commit in refs_commits |
| if to_semantic_version_tuple(ref) |
| ) |
| assert semantic_versions_commits, "failed to resolve commit to roll" |
| greatest_version, greatest_commit = semantic_versions_commits[-1] |
| presentation.logs["greatest semantic version"] = "v%d.%d.%d" % greatest_version |
| return greatest_commit |
| |
| |
| def GenTests(api): |
| def properties(**kwargs): |
| props = dict( |
| project="integration", |
| manifest="minimal", |
| remote="https://fuchsia.googlesource.com/integration", |
| import_in="stem", |
| import_from="fuchsia", |
| owners=["nobody@google.com", "noreply@google.com"], |
| ) |
| props.update(**kwargs) |
| return api.properties(**props) |
| |
| def ls_remote(greatest_release_revision="123abd"): |
| return api.step_data( |
| "edit manifest.resolve new revision.ls-remote", |
| api.raw_io.stream_output_text( |
| "\n".join( |
| [ |
| "a21bf2e6466095c7a2cdb991017da9639cf496e5 refs/tags/v2.6.0", |
| f"{greatest_release_revision} refs/tags/v2.10.1", |
| "419c94b6e07f93610e5124a0be0a7ee7140006ed refs/tags/v2.6.1rc1", |
| "e8ae137c96444ea313485ed1118c5e43b2099cf1 foo/bar/baz", |
| "e8ae137c96444ea313485ed1118c5e43b2099cf2 refs/tags/v1.2.3.4", |
| ] |
| ) |
| ), |
| ) |
| |
| yield ( |
| api.test("successful") |
| + properties() |
| + ls_remote() |
| + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # If the repo is already at the greatest release, the recipe should not attempt |
| # a roll. |
| yield ( |
| api.test("noop") |
| + properties() |
| + ls_remote("123abc") |
| + api.step_data( |
| "jiri project (2)", |
| api.jiri.project( |
| [ |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| }, |
| ] |
| ), |
| ) |
| ) |
| |
| # If the currently pinned revision and the main revision are not equal, |
| # but the gitiles log between them is empty, that probably means that the |
| # currently pinned revision is actually *ahead* of main, according to |
| # `gitiles log`. This sometimes happens, possibly due to GoB |
| # synchronization issues, and we should fail instead of rolling backwards. |
| yield ( |
| api.test("backwards_roll", status="INFRA_FAILURE") |
| + properties() |
| + ls_remote() |
| + api.gitiles.log("edit manifest.log", "A", n=0) |
| ) |