| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling Fuchsia layers into upper layers.""" |
| |
| from urlparse import urlparse |
| |
| from recipe_engine.config import Enum, List |
| from recipe_engine.recipe_api import Property |
| |
| # ROLL_TYPES lists the types of rolls we can perform on the target manifest. |
| # * 'import': An <import> tag will be updated. |
| # * 'project': A <project> tag will be updated. |
| ROLL_TYPES = ["import", "project"] |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/gerrit", |
| "fuchsia/gitiles", |
| "fuchsia/jiri", |
| "fuchsia/sso", |
| "fuchsia/status_check", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/python", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "project": Property(kind=str, help="Jiri remote manifest project", default=None), |
| "manifest": Property(kind=str, help="Jiri manifest to use"), |
| "remote": Property(kind=str, help="Remote manifest repository"), |
| "roll_type": Property( |
| kind=Enum(*ROLL_TYPES), help="The type of roll to perform", default="import" |
| ), |
| "import_in": Property( |
| kind=str, help="Path to the manifest to edit relative to $project" |
| ), |
| "import_from": Property( |
| kind=str, help="Name of the <project> or <import> to edit in $import_in" |
| ), |
| "lockfiles": Property( |
| kind=List(str), |
| default=(), |
| help=("The list of lockfiles to update in " '"${manifest}=${lockfile}" format'), |
| ), |
| # TODO: delete this property after we have full lockfile support in |
| # integration repo |
| "enforce_locks": Property( |
| kind=bool, default=False, help="Whether to enforce locks from lockfiles" |
| ), |
| "force_submit": Property( |
| kind=bool, |
| default=False, |
| help="Whether to force-submit the change, bypassing CQ", |
| ), |
| "dry_run": Property( |
| kind=bool, |
| default=False, |
| help=("Whether to dry-run the auto-roller (CQ+1 and abandon the change)"), |
| ), |
| "commit_divider": Property( |
| kind=str, |
| default="", |
| help=("Line of text to divide the commit header and body from the footers"), |
| ), |
| "send_comment": Property( |
| kind=bool, |
| default=True, |
| help="Whether to comment on the rolled commits once roll is complete", |
| ), |
| "owners": Property( |
| kind=List(str), |
| default=(), |
| help=( |
| "The owners responsible for watching this roller " |
| '(example: "username@google.com").' |
| ), |
| ), |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {header} |
| {body} |
| {divider} |
| {comment_footers}""" |
| |
| COMMIT_HEADER = """{project} {old}..{new} ({count} commits)""" |
| SINGLE_COMMIT_HEADER = """{project} {original_header}""" |
| SINGLE_COMMIT_BODY = """{original_body} |
| Original-Revision: {original_revision}""" |
| |
| # These footers must be set for the roll_comment recipe to be able to comment |
| # on the rolled CLs. |
| COMMENT_FOOTERS = """Rolled-Repo: {repo} |
| Rolled-Commits: {old}..{new}""" |
| |
| # If we're embedding the original commit message, prepend 'Original-' to lines |
| # which begin with these tags. |
| ESCAPE_TAGS = [ |
| "Reviewed-on:", |
| "Bug:", |
| "Fixed:", |
| ] |
| # If we're embedding the original commit message, remove lines which contain |
| # these tags. |
| FILTER_TAGS = [ |
| "Change-Id:", |
| "Reviewed-by:", |
| "Signed-off-by:", |
| "Acked-by:", |
| "CC:", |
| "Tested-by:", |
| "Commit-Queue:", |
| "Testability-Review:", |
| "API-Review:", |
| ] |
| |
| |
| def write_commit_message( |
| api, |
| log, |
| roll_from_repo, |
| import_from, |
| old_rev, |
| new_rev, |
| origin_host, |
| commit_divider, |
| send_comment, |
| ): |
| commit_lines = [] |
| |
| with api.step.nest("get gerrit change numbers"): |
| for commit in log: |
| commit_label = commit["id"][:7] |
| |
| if origin_host: |
| step = api.gerrit.change_details( |
| "change details for %s" % commit_label, |
| commit["id"], |
| host=origin_host, |
| test_data=api.json.test_api.output({"_number": 12345}), |
| ok_ret=(0, 1), |
| ) |
| if step.retcode == 0: |
| change = step.json.output |
| # url.path contains a leading '/'. |
| project_repo = urlparse(roll_from_repo).path |
| commit_label = "{label}:https://{gerrit_host}/c{project_repo}/+/{change_number}".format( |
| label=commit_label, |
| gerrit_host=origin_host, |
| project_repo=project_repo, |
| change_number=str(change["_number"]), |
| ) |
| commit_lines.append( |
| "{commit} {subject}".format( |
| commit=commit_label, subject=commit["message"].splitlines()[0], |
| ) |
| ) |
| |
| # If we're rolling a single change, replace the roll header and body with |
| # a sanitized version of the original change's commit message. |
| if len(log) == 1: |
| commit = log[0] |
| message_lines = commit["message"].splitlines() |
| message_lines = _sanitize_lines(message_lines) |
| header = SINGLE_COMMIT_HEADER.format( |
| project=import_from, original_header=message_lines[0], |
| ) |
| body = SINGLE_COMMIT_BODY.format( |
| original_body="\n".join(message_lines[1:]), original_revision=commit["id"], |
| ) |
| else: |
| header = COMMIT_HEADER.format( |
| project=import_from, old=old_rev[:7], new=new_rev[:7], count=len(log), |
| ) |
| body = "\n" + "\n".join(commit_lines) |
| |
| comment_footers = "" |
| if send_comment: |
| comment_footers = COMMENT_FOOTERS.format( |
| # For some reason gitiles occasionally considers 7-character hashes for |
| # fuchsia.git to be ambiguous, so we use longer ones here (see |
| # http://b/148289050). |
| old=old_rev[:14], |
| new=new_rev[:14], |
| repo=roll_from_repo, |
| ) |
| |
| message = COMMIT_MESSAGE.format( |
| header=header, |
| body=body, |
| divider=commit_divider, |
| comment_footers=comment_footers, |
| ) |
| return message |
| |
| |
| def _resolve_new_revision(api, repo): |
| input_commit = api.buildbucket.build.input.gitiles_commit |
| if input_commit.id: |
| commit_repo = "https://%s/%s" % (input_commit.host, input_commit.project) |
| assert commit_repo == repo, ( |
| "roll triggered by %s, but can only roll commits from %s" |
| % (commit_repo, repo) |
| ) |
| return input_commit.id |
| return api.gitiles.refs(repo).get("refs/heads/master", None) |
| |
| |
| def _sanitize_lines(lines): |
| """Sanitize lines of a commit message. |
| |
| Prepend 'Original-' to lines which begin with ESCAPE_TAGS. Filter |
| out lines which begin with FILTER_TAGS. |
| """ |
| lowercase_filter_tags = [tag.lower() for tag in FILTER_TAGS] |
| sanitized_lines = [] |
| for line in lines: |
| lowercase_line = line.lower() |
| if any((line.startswith(tag) for tag in ESCAPE_TAGS)): |
| line = "Original-" + line |
| elif any((tag in lowercase_line for tag in lowercase_filter_tags)): |
| continue |
| sanitized_lines.append(line) |
| return sanitized_lines |
| |
| |
| # This recipe has two 'modes' of operation: production and dry-run. Which mode |
| # of execution should be used is dictated by the 'dry_run' property. |
| # |
| # The purpose of dry-run mode is to test the auto-roller end-to-end. This is |
| # useful because now we can have an auto-roller in staging, and we can block |
| # updates behind 'dry_run' as a sort of feature gate. It is passed to |
| # api.auto_roller.attempt_roll() which handles committing changes. |
| def RunSteps( |
| api, |
| project, |
| manifest, |
| remote, |
| roll_type, |
| import_in, |
| import_from, |
| lockfiles, |
| enforce_locks, |
| force_submit, |
| dry_run, |
| commit_divider, |
| send_comment, |
| owners, |
| ): |
| with api.context(infra_steps=True): |
| if owners: |
| owners_step = api.step("owners", None) |
| owners_step.presentation.step_summary_text = ", ".join(owners) |
| |
| with api.step.nest("checkout"): |
| api.jiri.init(use_lock_file=enforce_locks) |
| api.jiri.import_manifest(manifest, remote, name=project) |
| api.jiri.update(run_hooks=False) |
| |
| project_dir = api.path["start_dir"].join(*project.split("/")) |
| with api.step.nest("edit manifest"), api.context(cwd=project_dir): |
| # Read the remote URL of the repo we're rolling from. |
| origin_manifest_element = api.jiri.read_manifest_element( |
| manifest=import_in, element_type=roll_type, element_name=import_from, |
| ) |
| |
| roll_from_repo = origin_manifest_element.get("remote") |
| if not roll_from_repo: |
| raise api.step.StepFailure("manifest element missing 'remote' field") |
| |
| roll_from_repo = api.sso.sso_to_https(roll_from_repo) |
| |
| old_rev = origin_manifest_element["revision"] |
| new_rev = _resolve_new_revision(api, roll_from_repo) |
| assert new_rev, "failed to resolve commit to roll" |
| |
| if old_rev == new_rev: |
| api.python.succeeding_step("nothing to roll", "manifest up-to-date") |
| return api.auto_roller.nothing_to_roll() |
| |
| log = api.gitiles.log( |
| roll_from_repo, "%s..%s" % (old_rev, new_rev), step_name="log" |
| ) |
| if not log: |
| api.python.infra_failing_step( |
| "detected backwards roll", |
| "expected %s to precede %s in git history" |
| % (old_rev[:7], new_rev[:7]), |
| ) |
| |
| # Determine whether to update manifest imports or projects. |
| if roll_type == "import": |
| imports = [(import_from, new_rev)] |
| projects = None |
| elif roll_type == "project": |
| imports = None |
| projects = [(import_from, new_rev)] |
| |
| api.jiri.edit_manifest(import_in, projects=projects, imports=imports) |
| for lock_entry in lockfiles: |
| fields = lock_entry.split("=") |
| manifest = fields[0] |
| lock = fields[1] |
| api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest]) |
| |
| origin_url = origin_manifest_element.get("gerrithost") |
| origin_host = None |
| if origin_url: |
| origin_host = urlparse(origin_url).netloc |
| assert origin_host, "gerrithost returned by jiri is not a valid URL: %r" % ( |
| origin_url |
| ) |
| |
| commit_message = write_commit_message( |
| api, |
| log=log, |
| roll_from_repo=roll_from_repo, |
| import_from=import_from, |
| old_rev=old_rev, |
| new_rev=new_rev, |
| origin_host=origin_host, |
| commit_divider=commit_divider, |
| send_comment=send_comment, |
| ) |
| |
| # If we're rolling a single commit, override the author with the original |
| # change's author, except prepend fuchsia.infra.roller. onto the email |
| # domain, so we don't truly attribute the change to the author. |
| author_override = None |
| if len(log) == 1: |
| author_override = log[0]["author"] |
| username, domain = author_override["email"].split("@") |
| author_override["email"] = "%s@fuchsia.infra.roller.%s" % (username, domain) |
| |
| # Land the changes. |
| change = api.auto_roller.attempt_roll( |
| api.gerrit.host_from_remote_url(remote), |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=commit_message, |
| force_submit=force_submit, |
| author_override=author_override, |
| dry_run=dry_run, |
| raise_on_failure=False, |
| ) |
| |
| return api.auto_roller.raw_result(change) |
| |
| |
| def GenTests(api): |
| DEFAULT_NEW_REV = "fc4dc762688d2263b254208f444f5c0a4b91bc07" |
| |
| def properties(**kwargs): |
| props = dict( |
| project="integration", |
| manifest="minimal", |
| remote="https://fuchsia.googlesource.com/integration", |
| import_in="stem", |
| import_from="fuchsia", |
| owners=["nobody@google.com", "noreply@google.com"], |
| ) |
| props.update(**kwargs) |
| return api.properties(**props) |
| |
| def bb_input(revision=DEFAULT_NEW_REV): |
| return api.buildbucket.ci_build( |
| git_repo="https://fuchsia.googlesource.com/fuchsia", revision=revision |
| ) |
| |
| def read_manifest_element(output): |
| return api.jiri.read_manifest_element( |
| element_name="fuchsia", |
| nesting="edit manifest", |
| test_output=output, |
| # No-ops, so just leave them unset. |
| manifest="", |
| element_type="", |
| ) |
| |
| # If available, we should always roll to the commit that triggered this build. |
| yield ( |
| api.status_check.test("resolve_revision_from_bb_input") |
| + properties() |
| + bb_input() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # If there is no buildbucket input, fall back to using gitiles to resolve |
| # the master revision. |
| yield ( |
| api.status_check.test("resolve_revision_from_gitiles") |
| + properties() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| # We should fall back to using gitiles to resolve the revision. |
| + api.gitiles.refs( |
| "edit manifest.refs", ("refs/heads/master", DEFAULT_NEW_REV), |
| ) |
| + api.gitiles.log("edit manifest.log", "A", add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # Test when the project to roll from is missing a 'remote' manifest attribute. |
| yield ( |
| api.status_check.test("missing_manifest_project_remote", status="failure") |
| + properties() |
| + bb_input() |
| + read_manifest_element({"revision": "123abc"}) |
| ) |
| |
| yield ( |
| api.status_check.test("successful_project_roll") |
| + properties(roll_type="project") |
| + bb_input() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # Test a successful single-commit roll. |
| yield ( |
| api.status_check.test("single_commit") |
| + properties(commit_divider="BEGIN_FOOTER") |
| + bb_input() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "gerrithost": "https://origin-host-review.googlesource.com", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log( |
| "edit manifest.log", |
| "A", |
| n=1, |
| add_change_id=True, |
| extra_footers={ |
| "Reviewed-by": "reviewer@foo.com", |
| "Reviewed-on": "https://fuchsia-review.googlesource.com/fuchsia", |
| "Tested-by": "tester@foo.com", |
| "Signed-off-by": "signer@foo.com", |
| "CC": "watcher@foo.com", |
| "Commit-Queue": "committer@foo.com", |
| "Testability-Review": "reviewer@foo.com", |
| "API-Review": "reviewer@foo.com", |
| "Bug": "12345", |
| "Fixed": "12345", |
| }, |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| # Test a successful roll of fuchsia into integration. |
| yield ( |
| api.status_check.test("roll_from_non_https_remote") |
| + properties() |
| + bb_input() |
| + read_manifest_element( |
| { |
| "remote": "sso://fuchsia/fuchsia", |
| "gerrithost": "https://fuchsia-review.googlesource.com", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # Test a no-op roll of fuchsia into integration (fuchsia is already pinned |
| # to the target revision). The recipe should exit early without attempting |
| # a roll. |
| yield ( |
| api.status_check.test("fuchsia_noop") |
| + properties() |
| + bb_input(revision=DEFAULT_NEW_REV) |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": DEFAULT_NEW_REV, |
| }, |
| ) |
| ) |
| |
| # Test a dry-run of the auto-roller for rolling fuchsia into integration. We |
| # substitute in mock data for the first check that the CQ dry-run completed |
| # by unsetting the CQ label to indicate that the CQ dry-run finished. |
| yield ( |
| api.status_check.test("fuchsia_dry_run") |
| + properties(dry_run=True) |
| + bb_input() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", add_change_id=True) |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| # Test a successful roll of fuchsia into integration with lockfile. |
| yield ( |
| api.status_check.test("fuchsia_with_lockfile") |
| + properties(lockfiles=["minimal=jiri.lock"]) |
| + bb_input() |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "123abc", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", add_change_id=True) |
| + api.auto_roller.success() |
| ) |
| |
| # If the currently pinned revision and the master revision are not equal, |
| # but the gitiles log between them is empty, that probably means that the |
| # currently pinned revision is actually *ahead* of master, according to |
| # `gitiles log`. This sometimes happens, possibly due to GoB |
| # synchronization issues, and we should fail instead of rolling backwards. |
| yield ( |
| api.status_check.test("backwards_roll", status="infra_failure") |
| + properties() |
| + bb_input(revision="oldhash") |
| + read_manifest_element( |
| { |
| "remote": "https://fuchsia.googlesource.com/fuchsia", |
| "revision": "newhash", |
| } |
| ) |
| + api.gitiles.log("edit manifest.log", "A", n=0) |
| ) |