| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling Fuchsia layers into upper layers.""" |
| |
| from urllib.parse import urlparse |
| |
| import re |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2 |
| from PB.recipes.fuchsia.fuchsia_roller import InputProperties |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/checkout", |
| "fuchsia/gerrit", |
| "fuchsia/git", |
| "fuchsia/gitiles", |
| "fuchsia/jiri", |
| "fuchsia/sso", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/json", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| # A newline is required between the divider and the real footers. Otherwise the |
| # divider will be in the last paragraph of the commit message, which will break |
| # Gerrit commit-msg hooks that check for an existing Change-Id footer, and don't |
| # consider footers in the last paragraph if it contains any non-footer lines. |
| COMMIT_MESSAGE = """[roll] Roll {header} |
| {body} |
| {divider} |
| |
| {comment_footers}""" |
| |
| COMMIT_HEADER = """{project} {old}..{new} ({count} commits)""" |
| SINGLE_COMMIT_HEADER = """{project} {original_header}""" |
| SINGLE_COMMIT_BODY = """{original_body} |
| Original-Revision: {original_revision}""" |
| |
| # These footers must be set for the roll_comment recipe to be able to comment |
| # on the rolled CLs. |
| COMMENT_FOOTERS = """Rolled-Repo: {repo} |
| Rolled-Commits: {old}..{new}""" |
| |
| # If we're embedding the original commit message, prepend 'Original-' to lines |
| # which begin with these tags. |
| ESCAPE_TAGS = [ |
| "Reviewed-on:", |
| "Bug:", |
| "Fixed:", |
| ] |
| # If we're embedding the original commit message, remove lines which contain |
| # these tags. |
| FILTER_TAGS = [ |
| "Change-Id:", |
| "Reviewed-by:", |
| "Signed-off-by:", |
| "Acked-by:", |
| "CC:", |
| "Tested-by:", |
| "Commit-Queue:", |
| "Testability-Review:", |
| "API-Review:", |
| "Auto-Submit:", |
| "Fuchsia-Auto-Submit:", |
| ] |
| |
| |
| # This recipe has two 'modes' of operation: production and dry-run. Which mode |
| # of execution should be used is dictated by the 'dry_run' property. |
| # |
| # The purpose of dry-run mode is to test the auto-roller end-to-end. This is |
| # useful because now we can have an auto-roller in staging, and we can block |
| # updates behind 'dry_run' as a sort of feature gate. It is passed to |
| # api.auto_roller.attempt_roll() which handles committing changes. |
| def RunSteps(api, props): |
| with api.context(infra_steps=True): |
| if props.roll_options.roller_owners: |
| api.step.empty( |
| "owners", step_text=", ".join(props.roll_options.roller_owners) |
| ) |
| |
| checkout = api.checkout.fuchsia_with_options( |
| manifest=props.manifest, |
| remote=props.remote, |
| project=props.project, |
| # Ignore the build input; we should always check out the manifest |
| # repository at HEAD before updating the manifest to reduce the |
| # likelihood of merge conflicts. |
| build_input=build_pb2.Build.Input(), |
| fetch_packages=False, |
| use_lock_file=True, |
| ) |
| project_path = checkout.project_path(props.project) |
| |
| with api.context(cwd=project_path): |
| with api.step.nest("edit manifest"): |
| # Read the remote URL of the repo we're rolling from. |
| origin_manifest_element = api.jiri.read_manifest_element( |
| manifest=props.import_in, |
| element_type="project", |
| element_name=props.import_from, |
| ) |
| |
| roll_from_repo = origin_manifest_element.get("remote") |
| if not roll_from_repo: |
| raise api.step.StepFailure("manifest element missing 'remote' field") |
| |
| roll_from_repo = api.sso.sso_to_https(roll_from_repo) |
| |
| old_rev = origin_manifest_element["revision"] |
| new_rev = resolve_new_revision(api, roll_from_repo, props.default_ref) |
| assert new_rev, "failed to resolve commit to roll" |
| |
| if old_rev == new_rev: |
| api.step.empty("nothing to roll", step_text="manifest up-to-date") |
| return api.auto_roller.nothing_to_roll() |
| |
| log = api.gitiles.log( |
| roll_from_repo, f"{old_rev}..{new_rev}", step_name="log" |
| ) |
| if not log: |
| api.step.empty( |
| "detected backwards roll", |
| step_text=f"expected {old_rev[:7]} to precede {new_rev[:7]} in git history", |
| status=api.step.INFRA_FAILURE, |
| ) |
| |
| api.jiri.edit_manifest( |
| props.import_in, projects=[(props.import_from, new_rev)] |
| ) |
| |
| origin_url = origin_manifest_element.get("gerrithost") |
| origin_host = None |
| if origin_url: |
| origin_host = urlparse(origin_url).netloc |
| assert ( |
| origin_host |
| ), f"gerrithost returned by jiri is not a valid URL: {origin_url!r}" |
| |
| commit_message = write_commit_message( |
| api, |
| log=log, |
| roll_from_repo=roll_from_repo, |
| import_from=props.import_from, |
| old_rev=old_rev, |
| new_rev=new_rev, |
| origin_host=origin_host, |
| commit_divider=props.commit_divider, |
| send_comment=props.send_comment, |
| ) |
| |
| # If we're rolling a single commit, override the author with the original |
| # change's author, except prepend fuchsia.infra.roller. onto the email |
| # domain, so we don't truly attribute the change to the author. |
| author_override = None |
| if len(log) == 1: |
| author_override = log[0]["author"] |
| try: |
| username, domain = author_override["email"].split("@") |
| except ValueError: |
| # Don't override the author if the committer email is invalid. |
| # The commit email is user-configured and not guaranteed to be |
| # well-formed. |
| author_override = None |
| else: |
| author_override["email"] = f"{username}@fuchsia.infra.roller.{domain}" |
| |
| # Land the changes. |
| change = api.auto_roller.attempt_roll( |
| props.roll_options, |
| repo_dir=project_path, |
| commit_message=commit_message, |
| author_override=author_override, |
| raise_on_failure=False, |
| ) |
| |
| return api.auto_roller.raw_result(change) |
| |
| |
| def write_commit_message( |
| api, |
| log, |
| roll_from_repo, |
| import_from, |
| old_rev, |
| new_rev, |
| origin_host, |
| commit_divider, |
| send_comment, |
| ): |
| commit_lines = [] |
| |
| with api.step.nest("get gerrit change numbers"): |
| for commit in log: |
| commit_label = commit["id"][:7] |
| |
| if origin_host: |
| step = api.gerrit.change_details( |
| f"change details for {commit_label}", |
| commit["id"], |
| host=origin_host, |
| test_data=api.json.test_api.output({"_number": 12345}), |
| ok_ret=(0, 1), |
| ) |
| if step.retcode == 0: |
| change = step.json.output |
| # url.path contains a leading '/'. |
| project_repo = urlparse(roll_from_repo).path |
| commit_label = f"{commit_label}:https://{origin_host}/c{project_repo}/+/{str(change['_number'])}" |
| commit_lines.append(f"{commit_label} {commit['message'].splitlines()[0]}") |
| |
| # If we're rolling a single change, replace the roll header and body with |
| # a sanitized version of the original change's commit message. |
| if len(log) == 1: |
| commit = log[0] |
| message_lines = commit["message"].splitlines() |
| message_lines = sanitize_lines(message_lines) |
| header = SINGLE_COMMIT_HEADER.format( |
| project=import_from, |
| original_header=message_lines[0], |
| ) |
| body = SINGLE_COMMIT_BODY.format( |
| original_body="\n".join(message_lines[1:]), |
| original_revision=commit["id"], |
| ) |
| else: |
| header = COMMIT_HEADER.format( |
| project=import_from, |
| old=old_rev[:7], |
| new=new_rev[:7], |
| count=len(log), |
| ) |
| body = "\n" + "\n".join(commit_lines) |
| |
| comment_footers = "" |
| if send_comment: |
| comment_footers = COMMENT_FOOTERS.format( |
| # For some reason gitiles occasionally considers 7-character hashes for |
| # fuchsia.git to be ambiguous, so we use longer ones here (see |
| # http://b/148289050). |
| old=old_rev[:14], |
| new=new_rev[:14], |
| repo=roll_from_repo, |
| ) |
| |
| message = COMMIT_MESSAGE.format( |
| header=header, |
| body=body, |
| divider=commit_divider, |
| comment_footers=comment_footers, |
| ) |
| return message |
| |
| |
| def resolve_new_revision(api, repo, default_ref): |
| input_commit = api.buildbucket.build.input.gitiles_commit |
| if ( |
| input_commit.id |
| and f"https://{input_commit.host}/{input_commit.project}" == repo |
| ): |
| return input_commit.id |
| assert ( |
| default_ref |
| ), "`default_ref` property is required when there's no input commit" |
| return api.git.get_remote_branch_head(repo, default_ref) |
| |
| |
| def sanitize_lines(lines): |
| """Sanitize lines of a commit message. |
| |
| Prepend 'Original-' to lines which begin with ESCAPE_TAGS. Filter |
| out lines which begin with FILTER_TAGS. |
| """ |
| lowercase_filter_tags = [tag.lower() for tag in FILTER_TAGS] |
| sanitized_lines = [] |
| for line in lines: |
| lowercase_line = line.lower() |
| # Rolls and relands contain the original commit message with each line |
| # prefixed with "> " (likewise, a reland of a reland will contain |
| # double-prefixed lines). Bugdroid disregards the prefix when finding |
| # bugs to comment on, so to prevent it from adding comments for rolls |
| # of revert/reland CLs, we should also escape >-prefixed tags. |
| prefix_match = re.match(r"(> )*", line) |
| tag_start = prefix_match.end() |
| if any((line.startswith(tag, tag_start) for tag in ESCAPE_TAGS)): |
| line = prefix_match.group() + "Original-" + line[prefix_match.end() :] |
| elif any((tag in lowercase_line for tag in lowercase_filter_tags)): |
| continue |
| sanitized_lines.append(line) |
| return sanitized_lines |
| |
| |
| def GenTests(api): |
| DEFAULT_NEW_REV = "fc4dc762688d2263b254208f444f5c0a4b91bc07" |
| |
| def properties(dry_run=False, **kwargs): |
| props = dict( |
| project="integration", |
| manifest="minimal", |
| remote="https://fuchsia.googlesource.com/integration", |
| import_in="stem", |
| import_from="fuchsia", |
| default_ref="refs/heads/main", |
| send_comment=True, |
| roll_options=api.auto_roller.Options( |
| remote="https://fuchsia.googlesource.com/integration", |
| roller_owners=["nobody@google.com", "noreply@google.com"], |
| dry_run=dry_run, |
| ), |
| ) |
| props.update(**kwargs) |
| return api.properties(**props) |
| |
| def test(name, **kwargs): |
| kwargs.setdefault("revision", DEFAULT_NEW_REV) |
| return api.buildbucket_util.test(name, **kwargs) |
| |
| def read_manifest_element(output): |
| return api.jiri.read_manifest_element( |
| element_name="fuchsia", |
| nesting="edit manifest", |
| test_output=output, |
| ) |
| |
| # If available, we should always roll to the commit that triggered this build. |
| yield ( |
| test("resolve_revision_from_bb_input") |
| + properties() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # If there is no buildbucket input, fall back to using gitiles to resolve |
| # HEAD. |
| yield ( |
| api.test("resolve_revision_from_gitiles") |
| + properties() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| # We should fall back to using gitiles to resolve the revision. |
| + api.git.get_remote_branch_head("edit manifest.git ls-remote", DEFAULT_NEW_REV) |
| + api.gitiles.log("edit manifest.log", "A", add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # Test when the project to roll from is missing a 'remote' manifest attribute. |
| yield ( |
| test("missing_manifest_project_remote", status="FAILURE") |
| + properties() |
| + read_manifest_element({"revision": "123abc"}) |
| ) |
| |
| yield ( |
| test("successful_project_roll") |
| + properties() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # Test a successful single-commit roll. |
| yield ( |
| test("single_commit") |
| + properties(commit_divider="BEGIN_FOOTER") |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "gerrithost": "https://origin-host-review.googlesource.com", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log( |
| "edit manifest.log", |
| "A", |
| n=1, |
| add_change_id=True, |
| extra_footers={ |
| "Reviewed-by": "reviewer@foo.com", |
| "Reviewed-on": "https://fuchsia-review.googlesource.com/fuchsia", |
| "Tested-by": "tester@foo.com", |
| "Signed-off-by": "signer@foo.com", |
| "CC": "watcher@foo.com", |
| "Commit-Queue": "committer@foo.com", |
| "Testability-Review": "reviewer@foo.com", |
| "API-Review": "reviewer@foo.com", |
| "Bug": "12345", |
| "Fixed": "12345", |
| # This line should be handled as if it's part of the quoted |
| # original commit message for a (double) revert/reland. |
| "> > Fixed": "456", |
| }, |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| # The roller should not break if the committer email is invalid. |
| yield ( |
| test("single_commit_invalid_email") |
| + properties(commit_divider="BEGIN_FOOTER") |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "gerrithost": "https://origin-host-review.googlesource.com", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log( |
| "edit manifest.log", |
| "A", |
| n=1, |
| # Email is invalid - contains multiple @ signs. |
| committer_email="invalid@invalid@invalid.com", |
| add_change_id=True, |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| # Test a successful roll of fuchsia into integration. |
| yield ( |
| test("roll_from_non_https_remote") |
| + properties() |
| + read_manifest_element( |
| { |
| "remote": "sso://fuchsia/fuchsia", |
| "gerrithost": "https://fuchsia-review.googlesource.com", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # Test a no-op roll of fuchsia into integration (fuchsia is already pinned |
| # to the target revision). The recipe should exit early without attempting |
| # a roll. |
| yield ( |
| test("fuchsia_noop", revision=DEFAULT_NEW_REV) |
| + properties() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": DEFAULT_NEW_REV, |
| }, |
| ) |
| ) |
| |
| # Test a dry-run of the auto-roller for rolling fuchsia into integration. We |
| # substitute in mock data for the first check that the CQ dry-run completed |
| # by unsetting the CQ label to indicate that the CQ dry-run finished. |
| yield ( |
| test("fuchsia_dry_run") |
| + properties(dry_run=True) |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", add_change_id=True) |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| # If the currently pinned revision and HEAD are not equal, but the gitiles |
| # log between them is empty, that probably means that the currently pinned |
| # revision is actually *ahead* of HEAD, according to `gitiles log`. This |
| # sometimes happens, possibly due to GoB synchronization issues, and we |
| # should fail instead of rolling backwards. |
| yield ( |
| test("backwards_roll", revision="oldhash", status="INFRA_FAILURE") |
| + properties() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "newhash", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", n=0) |
| ) |