recipes/fuchsia_roller.py

# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for rolling Fuchsia layers into upper layers."""

from urllib.parse import urlparse

from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2
from PB.recipes.fuchsia.fuchsia_roller import InputProperties

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/git",
    "fuchsia/git_roll_util",
    "fuchsia/gitiles",
    "fuchsia/jiri",
    "fuchsia/sso",
    "recipe_engine/context",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = InputProperties


# This recipe has two 'modes' of operation: production and dry-run. Which mode
# of execution should be used is dictated by the 'dry_run' property.
#
# The purpose of dry-run mode is to test the auto-roller end-to-end. This is
# useful because now we can have an auto-roller in staging, and we can block
# updates behind 'dry_run' as a sort of feature gate. It is passed to
# api.auto_roller.attempt_roll() which handles committing changes.
def RunSteps(api, props):
    with api.context(infra_steps=True):
        if props.roll_options.roller_owners:
            api.step.empty(
                "owners", step_text=", ".join(props.roll_options.roller_owners)
            )

    checkout = api.checkout.fuchsia_with_options(
        manifest=props.manifest,
        remote=props.remote,
        project=props.project,
        # Ignore the build input; we should always check out the manifest
        # repository at HEAD before updating the manifest to reduce the
        # likelihood of merge conflicts.
        build_input=build_pb2.Build.Input(),
        fetch_packages=False,
        use_lock_file=True,
    )
    project_path = checkout.project_path(props.project)

    with api.context(cwd=project_path):
        with api.step.nest("edit manifest"):
            # Read the remote URL of the repo we're rolling from.
            origin_manifest_element = api.jiri.read_manifest_element(
                manifest=props.import_in,
                element_type="project",
                element_name=props.import_from,
            )

            roll_from_repo = origin_manifest_element.get("remote")
            if not roll_from_repo:
                raise api.step.StepFailure("manifest element missing 'remote' field")

            roll_from_repo = api.sso.sso_to_https(roll_from_repo)

            old_rev = origin_manifest_element["revision"]
            new_rev = api.git_roll_util.resolve_new_revision(
                roll_from_repo, props.default_ref
            )
            assert new_rev, "failed to resolve commit to roll"

            if old_rev == new_rev:
                api.step.empty("nothing to roll", step_text="manifest up-to-date")
                return api.auto_roller.nothing_to_roll()

            api.jiri.edit_manifest(
                props.import_in, projects=[(props.import_from, new_rev)]
            )

        origin_url = origin_manifest_element.get("gerrithost")
        origin_host = None
        if origin_url:
            origin_host = urlparse(origin_url).netloc
            assert (
                origin_host
            ), f"gerrithost returned by jiri is not a valid URL: {origin_url!r}"

        commit_message, author_override = (
            api.git_roll_util.get_commit_message_and_author(
                repo_url=roll_from_repo,
                repo_short_name=props.import_from,
                old_rev=old_rev,
                new_rev=new_rev,
                gerrit_host=origin_host,
                commit_divider=props.commit_divider,
                send_comment=props.send_comment,
            )
        )

        # Land the changes.
        change = api.auto_roller.attempt_roll(
            props.roll_options,
            repo_dir=project_path,
            commit_message=commit_message,
            author_override=author_override,
            raise_on_failure=False,
        )

        return api.auto_roller.raw_result(change)


def GenTests(api):
    DEFAULT_NEW_REV = "fc4dc762688d2263b254208f444f5c0a4b91bc07"

    def properties(dry_run=False, **kwargs):
        props = dict(
            project="integration",
            manifest="minimal",
            remote="https://fuchsia.googlesource.com/integration",
            import_in="stem",
            import_from="fuchsia",
            default_ref="refs/heads/main",
            send_comment=True,
            roll_options=api.auto_roller.Options(
                remote="https://fuchsia.googlesource.com/integration",
                roller_owners=["nobody@google.com", "noreply@google.com"],
                dry_run=dry_run,
            ),
        )
        props.update(**kwargs)
        return api.properties(**props)

    def test(name, **kwargs):
        kwargs.setdefault("revision", DEFAULT_NEW_REV)
        return api.buildbucket_util.test(name, **kwargs)

    def read_manifest_element(output):
        return api.jiri.read_manifest_element(
            element_name="fuchsia",
            nesting="edit manifest",
            test_output=output,
        )

    # # If available, we should always roll to the commit that triggered this build.
    yield (
        test("resolve_revision_from_bb_input")
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "123abc",
            }
        )
        + api.gitiles.log("log", "A", add_change_id=True)
        + api.auto_roller.success()
    )

    # # If there is no buildbucket input, fall back to using gitiles to resolve
    # # HEAD.
    yield (
        api.test("resolve_revision_from_gitiles")
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "123abc",
            }
        )
        # We should fall back to using gitiles to resolve the revision.
        + api.git.get_remote_branch_head("edit manifest.git ls-remote", DEFAULT_NEW_REV)
        + api.gitiles.log("log", "A", add_change_id=True)
        + api.auto_roller.success()
    )

    # Test when the project to roll from is missing a 'remote' manifest attribute.
    yield (
        test("missing_manifest_project_remote", status="FAILURE")
        + properties()
        + read_manifest_element({"revision": "123abc"})
    )

    yield (
        test("successful_project_roll")
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "123abc",
            }
        )
        + api.gitiles.log("log", "A", n=2, add_change_id=True)
        + api.auto_roller.success()
    )

    # Test a successful single-commit roll.
    yield (
        test("single_commit")
        + properties(commit_divider="BEGIN_FOOTER")
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "gerrithost": "https://origin-host-review.googlesource.com",
                "revision": "123abc",
            }
        )
        + api.gitiles.log(
            "log",
            "A",
            n=1,
            add_change_id=True,
            extra_footers={
                "Reviewed-by": "reviewer@foo.com",
                "Reviewed-on": "https://fuchsia-review.googlesource.com/fuchsia",
                "Tested-by": "tester@foo.com",
                "Signed-off-by": "signer@foo.com",
                "CC": "watcher@foo.com",
                "Commit-Queue": "committer@foo.com",
                "Testability-Review": "reviewer@foo.com",
                "API-Review": "reviewer@foo.com",
                "Bug": "12345",
                "Fixed": "12345",
                # This line should be handled as if it's part of the quoted
                # original commit message for a (double) revert/reland.
                "> > Fixed": "456",
            },
        )
        + api.auto_roller.success()
    )

    # The roller should not break if the committer email is invalid.
    yield (
        test("single_commit_invalid_email")
        + properties(commit_divider="BEGIN_FOOTER")
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "gerrithost": "https://origin-host-review.googlesource.com",
                "revision": "123abc",
            }
        )
        + api.gitiles.log(
            "log",
            "A",
            n=1,
            # Email is invalid - contains multiple @ signs.
            committer_email="invalid@invalid@invalid.com",
            add_change_id=True,
        )
        + api.auto_roller.success()
    )

    # Test a successful roll of fuchsia into integration.
    yield (
        test("roll_from_non_https_remote")
        + properties()
        + read_manifest_element(
            {
                "remote": "sso://fuchsia/fuchsia",
                "gerrithost": "https://fuchsia-review.googlesource.com",
                "revision": "123abc",
            }
        )
        + api.gitiles.log("log", "A", n=2, add_change_id=True)
        + api.auto_roller.success()
    )

    # Test a no-op roll of fuchsia into integration (fuchsia is already pinned
    # to the target revision). The recipe should exit early without attempting
    # a roll.
    yield (
        test("fuchsia_noop", revision=DEFAULT_NEW_REV)
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": DEFAULT_NEW_REV,
            },
        )
    )

    # Test a dry-run of the auto-roller for rolling fuchsia into integration. We
    # substitute in mock data for the first check that the CQ dry-run completed
    # by unsetting the CQ label to indicate that the CQ dry-run finished.
    yield (
        test("fuchsia_dry_run")
        + properties(dry_run=True)
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "123abc",
            }
        )
        + api.gitiles.log("log", "A", add_change_id=True)
        + api.auto_roller.dry_run_success()
    )

    # If the currently pinned revision and HEAD are not equal, but the gitiles
    # log between them is empty, that probably means that the currently pinned
    # revision is actually *ahead* of HEAD, according to `gitiles log`. This
    # sometimes happens, possibly due to GoB synchronization issues, and we
    # should fail instead of rolling backwards.
    yield (
        test("backwards_roll", revision="oldhash", status="INFRA_FAILURE")
        + properties()
        + read_manifest_element(
            {
                "remote": "https://fuchsia.googlesource.com/fuchsia",
                "revision": "newhash",
            }
        )
        + api.gitiles.log("log", "A", n=0)
    )