recipes/fuchsia_roller.py

# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for rolling Fuchsia layers into upper layers."""

from future.moves.urllib.parse import urlparse

import re
from PB.recipes.fuchsia.fuchsia_roller import InputProperties

PYTHON_VERSION_COMPATIBILITY = "PY3"

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/gerrit",
    "fuchsia/git",
    "fuchsia/gitiles",
    "fuchsia/jiri",
    "fuchsia/sso",
    "fuchsia/status_check",
    "recipe_engine/buildbucket",
    "recipe_engine/context",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties

COMMIT_MESSAGE = """[roll] Roll {header}
{body}
{divider}
{comment_footers}"""

COMMIT_HEADER = """{project} {old}..{new} ({count} commits)"""
SINGLE_COMMIT_HEADER = """{project} {original_header}"""
SINGLE_COMMIT_BODY = """{original_body}
Original-Revision: {original_revision}"""

# These footers must be set for the roll_comment recipe to be able to comment
# on the rolled CLs.
COMMENT_FOOTERS = """Rolled-Repo: {repo}
Rolled-Commits: {old}..{new}"""

# If we're embedding the original commit message, prepend 'Original-' to lines
# which begin with these tags.
ESCAPE_TAGS = [
    "Reviewed-on:",
    "Bug:",
    "Fixed:",
]
# If we're embedding the original commit message, remove lines which contain
# these tags.
FILTER_TAGS = [
    "Change-Id:",
    "Reviewed-by:",
    "Signed-off-by:",
    "Acked-by:",
    "CC:",
    "Tested-by:",
    "Commit-Queue:",
    "Testability-Review:",
    "API-Review:",
    "Auto-Submit:",
    "Fuchsia-Auto-Submit:",
]


def write_commit_message(
    api,
    log,
    roll_from_repo,
    import_from,
    old_rev,
    new_rev,
    origin_host,
    commit_divider,
    send_comment,
):
    commit_lines = []

    with api.step.nest("get gerrit change numbers"):
        for commit in log:
            commit_label = commit["id"][:7]

            if origin_host:
                step = api.gerrit.change_details(
                    "change details for %s" % commit_label,
                    commit["id"],
                    host=origin_host,
                    test_data=api.json.test_api.output({"_number": 12345}),
                    ok_ret=(0, 1),
                )
                if step.retcode == 0:
                    change = step.json.output
                    # url.path contains a leading '/'.
                    project_repo = urlparse(roll_from_repo).path
                    commit_label = "{label}:https://{gerrit_host}/c{project_repo}/+/{change_number}".format(
                        label=commit_label,
                        gerrit_host=origin_host,
                        project_repo=project_repo,
                        change_number=str(change["_number"]),
                    )
            commit_lines.append(
                "{commit} {subject}".format(
                    commit=commit_label,
                    subject=commit["message"].splitlines()[0],
                )
            )

    # If we're rolling a single change, replace the roll header and body with
    # a sanitized version of the original change's commit message.
    if len(log) == 1:
        commit = log[0]
        message_lines = commit["message"].splitlines()
        message_lines = sanitize_lines(message_lines)
        header = SINGLE_COMMIT_HEADER.format(
            project=import_from,
            original_header=message_lines[0],
        )
        body = SINGLE_COMMIT_BODY.format(
            original_body="\n".join(message_lines[1:]),
            original_revision=commit["id"],
        )
    else:
        header = COMMIT_HEADER.format(
            project=import_from,
            old=old_rev[:7],
            new=new_rev[:7],
            count=len(log),
        )
        body = "\n" + "\n".join(commit_lines)

    comment_footers = ""
    if send_comment:
        comment_footers = COMMENT_FOOTERS.format(
            # For some reason gitiles occasionally considers 7-character hashes for
            # fuchsia.git to be ambiguous, so we use longer ones here (see
            # http://b/148289050).
            old=old_rev[:14],
            new=new_rev[:14],
            repo=roll_from_repo,
        )

    message = COMMIT_MESSAGE.format(
        header=header,
        body=body,
        divider=commit_divider,
        comment_footers=comment_footers,
    )
    return message


def resolve_new_revision(api, repo, default_ref):
    input_commit = api.buildbucket.build.input.gitiles_commit
    if (
        input_commit.id
        and "https://%s/%s" % (input_commit.host, input_commit.project) == repo
    ):
        return input_commit.id
    assert (
        default_ref
    ), "`default_ref` property is required when there's no input commit"
    return api.git.get_remote_branch_head(repo, default_ref)


def sanitize_lines(lines):
    """Sanitize lines of a commit message.

    Prepend 'Original-' to lines which begin with ESCAPE_TAGS. Filter
    out lines which begin with FILTER_TAGS.
    """
    lowercase_filter_tags = [tag.lower() for tag in FILTER_TAGS]
    sanitized_lines = []
    for line in lines:
        lowercase_line = line.lower()
        # Rolls and relands contain the original commit message with each line
        # prefixed with "> " (likewise, a reland of a reland will contain
        # double-prefixed lines). Bugdroid disregards the prefix when finding
        # bugs to comment on, so to prevent it from adding comments for rolls
        # of revert/reland CLs, we should also escape >-prefixed tags.
        prefix_match = re.match(r"(> )*", line)
        tag_start = prefix_match.end()
        if any((line.startswith(tag, tag_start) for tag in ESCAPE_TAGS)):
            line = prefix_match.group() + "Original-" + line[prefix_match.end() :]
        elif any((tag in lowercase_line for tag in lowercase_filter_tags)):
            continue
        sanitized_lines.append(line)
    return sanitized_lines


# This recipe has two 'modes' of operation: production and dry-run. Which mode
# of execution should be used is dictated by the 'dry_run' property.
#
# The purpose of dry-run mode is to test the auto-roller end-to-end. This is
# useful because now we can have an auto-roller in staging, and we can block
# updates behind 'dry_run' as a sort of feature gate. It is passed to
# api.auto_roller.attempt_roll() which handles committing changes.
def RunSteps(api, props):
    with api.context(infra_steps=True):
        assert props.roll_type in ("import", "project")
        if props.owners:
            api.step.empty("owners", step_text=", ".join(props.owners))

        with api.step.nest("checkout"):
            api.jiri.init(use_lock_file=props.enforce_locks)
            api.jiri.import_manifest(props.manifest, props.remote, name=props.project)
            api.jiri.update(run_hooks=False)

        project_dir = api.path["start_dir"].join(*props.project.split("/"))
        with api.step.nest("edit manifest"), api.context(cwd=project_dir):
            # Read the remote URL of the repo we're rolling from.
            origin_manifest_element = api.jiri.read_manifest_element(
                manifest=props.import_in,
                element_type=props.roll_type,
                element_name=props.import_from,
            )

            roll_from_repo = origin_manifest_element.get("remote")
            if not roll_from_repo:
                raise api.step.StepFailure("manifest element missing 'remote' field")

            roll_from_repo = api.sso.sso_to_https(roll_from_repo)

            old_rev = origin_manifest_element["revision"]
            new_rev = resolve_new_revision(api, roll_from_repo, props.default_ref)
            assert new_rev, "failed to resolve commit to roll"

            if old_rev == new_rev:
                api.step.empty("nothing to roll", step_text="manifest up-to-date")
                return api.auto_roller.nothing_to_roll()

            log = api.gitiles.log(
                roll_from_repo, "%s..%s" % (old_rev, new_rev), step_name="log"
            )
            if not log:
                api.step.empty(
                    "detected backwards roll",
                    step_text="expected %s to precede %s in git history"
                    % (old_rev[:7], new_rev[:7]),
                    status=api.step.INFRA_FAILURE,
                )

            # Determine whether to update manifest imports or projects.
            if props.roll_type == "import":
                imports = [(props.import_from, new_rev)]
                projects = None
            elif props.roll_type == "project":
                imports = None
                projects = [(props.import_from, new_rev)]

            api.jiri.edit_manifest(props.import_in, projects=projects, imports=imports)

        origin_url = origin_manifest_element.get("gerrithost")
        origin_host = None
        if origin_url:
            origin_host = urlparse(origin_url).netloc
            assert origin_host, "gerrithost returned by jiri is not a valid URL: %r" % (
                origin_url
            )

        commit_message = write_commit_message(
            api,
            log=log,
            roll_from_repo=roll_from_repo,
            import_from=props.import_from,
            old_rev=old_rev,
            new_rev=new_rev,
            origin_host=origin_host,
            commit_divider=props.commit_divider,
            send_comment=props.send_comment,
        )

        # If we're rolling a single commit, override the author with the original
        # change's author, except prepend fuchsia.infra.roller. onto the email
        # domain, so we don't truly attribute the change to the author.
        author_override = None
        if len(log) == 1:
            author_override = log[0]["author"]
            username, domain = author_override["email"].split("@")
            author_override["email"] = "%s@fuchsia.infra.roller.%s" % (username, domain)

        # Land the changes.
        change = api.auto_roller.attempt_roll(
            api.gerrit.host_from_remote_url(props.remote),
            gerrit_project=props.project,
            repo_dir=project_dir,
            commit_message=commit_message,
            force_submit=props.force_submit,
            no_tryjobs=props.no_tryjobs,
            author_override=author_override,
            cl_notify_option=props.cl_notify_option,
            create_unique_id=props.create_unique_change_id,
            dry_run=props.dry_run,
            raise_on_failure=False,
            roller_owners=props.owners,
            include_tryjobs=props.include_tryjobs,
        )

        return api.auto_roller.raw_result(change)


def GenTests(api):
    DEFAULT_NEW_REV = "fc4dc762688d2263b254208f444f5c0a4b91bc07"

    def properties(**kwargs):
        props = dict(
            project="integration",
            manifest="minimal",
            remote="https://fuchsia.googlesource.com/integration",
            import_in="stem",
            import_from="fuchsia",
            default_ref="refs/heads/main",
            owners=["nobody@google.com", "noreply@google.com"],
            roll_type="import",
            send_comment=True,
        )
        props.update(**kwargs)
        return api.properties(**props)

    def test(name, **kwargs):
        kwargs.setdefault("revision", DEFAULT_NEW_REV)
        return api.buildbucket_util.test(name, **kwargs)

    def read_manifest_element(output):
        return api.jiri.read_manifest_element(
            element_name="fuchsia",
            nesting="edit manifest",
            test_output=output,
            # No-ops, so just leave them unset.
            manifest="",
            element_type="",
        )

    # If available, we should always roll to the commit that triggered this build.
    yield (
        test("resolve_revision_from_bb_input")
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "123abc",
            }
        )
        + api.gitiles.log("edit manifest.log", "A", add_change_id=True)
        + api.auto_roller.success()
    )

    # If there is no buildbucket input, fall back to using gitiles to resolve
    # HEAD.
    yield (
        api.status_check.test("resolve_revision_from_gitiles")
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "123abc",
            }
        )
        # We should fall back to using gitiles to resolve the revision.
        + api.git.get_remote_branch_head("edit manifest.git ls-remote", DEFAULT_NEW_REV)
        + api.gitiles.log("edit manifest.log", "A", add_change_id=True)
        + api.auto_roller.success()
    )

    # Test when the project to roll from is missing a 'remote' manifest attribute.
    yield (
        test("missing_manifest_project_remote", status="failure")
        + properties()
        + read_manifest_element({"revision": "123abc"})
    )

    yield (
        test("successful_project_roll")
        + properties(roll_type="project", include_tryjobs={"bucket": ["builder"]})
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "123abc",
            }
        )
        + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True)
        + api.auto_roller.success()
    )

    # Test a successful single-commit roll.
    yield (
        test("single_commit")
        + properties(commit_divider="BEGIN_FOOTER")
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "gerrithost": "https://origin-host-review.googlesource.com",
                "revision": "123abc",
            }
        )
        + api.gitiles.log(
            "edit manifest.log",
            "A",
            n=1,
            add_change_id=True,
            extra_footers={
                "Reviewed-by": "reviewer@foo.com",
                "Reviewed-on": "https://fuchsia-review.googlesource.com/fuchsia",
                "Tested-by": "tester@foo.com",
                "Signed-off-by": "signer@foo.com",
                "CC": "watcher@foo.com",
                "Commit-Queue": "committer@foo.com",
                "Testability-Review": "reviewer@foo.com",
                "API-Review": "reviewer@foo.com",
                "Bug": "12345",
                "Fixed": "12345",
                # This line should be handled as if it's part of the quoted
                # original commit message for a (double) revert/reland.
                "> > Fixed": "456",
            },
        )
        + api.auto_roller.success()
    )

    # Test a successful roll of fuchsia into integration.
    yield (
        test("roll_from_non_https_remote")
        + properties()
        + read_manifest_element(
            {
                "remote": "sso://fuchsia/fuchsia",
                "gerrithost": "https://fuchsia-review.googlesource.com",
                "revision": "123abc",
            }
        )
        + api.gitiles.log("edit manifest.log", "A", n=2, add_change_id=True)
        + api.auto_roller.success()
    )

    # Test a no-op roll of fuchsia into integration (fuchsia is already pinned
    # to the target revision). The recipe should exit early without attempting
    # a roll.
    yield (
        test("fuchsia_noop", revision=DEFAULT_NEW_REV)
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": DEFAULT_NEW_REV,
            },
        )
    )

    # Test a dry-run of the auto-roller for rolling fuchsia into integration. We
    # substitute in mock data for the first check that the CQ dry-run completed
    # by unsetting the CQ label to indicate that the CQ dry-run finished.
    yield (
        test("fuchsia_dry_run")
        + properties(dry_run=True)
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "123abc",
            }
        )
        + api.gitiles.log("edit manifest.log", "A", add_change_id=True)
        + api.auto_roller.dry_run_success()
    )

    # If the currently pinned revision and HEAD are not equal, but the gitiles
    # log between them is empty, that probably means that the currently pinned
    # revision is actually *ahead* of HEAD, according to `gitiles log`. This
    # sometimes happens, possibly due to GoB synchronization issues, and we
    # should fail instead of rolling backwards.
    yield (
        test("backwards_roll", revision="oldhash", status="infra_failure")
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "newhash",
            }
        )
        + api.gitiles.log("edit manifest.log", "A", n=0)
    )