recipes/release/cherry_pick.py

# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Cherry-pick one or more changes onto an integration.git release branch.

Inputs
  - Project: The project to apply cherry-pick(s) to.
  - Change(s):
    - Revision: The revision to cherry-pick.
    - Change-Id: The Gerrit change ID to cherry-pick.

Options
  - Prepare-branch: Short-circuit after ensuring project's branch exists.
  - Dryrun: Run recipe without modifying anything remotely.
"""

from urllib.parse import urlparse

from RECIPE_MODULES.fuchsia.utils import pluralize

from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2
from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2
from PB.recipe_engine.result import RawResult
from PB.recipes.fuchsia.release.cherry_pick import InputProperties

DEPS = [
    "fuchsia/checkout",
    "fuchsia/gerrit",
    "fuchsia/git",
    "fuchsia/jiri",
    "fuchsia/release",
    "fuchsia/sso",
    "fuchsia/utils",
    "recipe_engine/context",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
]

PROPERTIES = InputProperties

COMMIT_MESSAGE = (
    "{prefix} Update {project} to {revision} after cherry-pick {change_urls}"
    "\n\n{commit_msgs}"
)


def RunSteps(api, props):
    with api.step.nest("check inputs"):
        if not props.changes:
            return RawResult(
                summary_markdown="`changes` must be specified",
                status=common_pb2.FAILURE,
            )
        if not props.target_project:
            return RawResult(
                summary_markdown="`target_project` must be specified",
                status=common_pb2.FAILURE,
            )
        if not props.target_remote:
            return RawResult(
                summary_markdown="`target_remote` must be specified",
                status=common_pb2.FAILURE,
            )
        if not api.release.validate_branch(props.target_branch):
            return RawResult(
                summary_markdown='`target_branch` must start with "releases/"',
                status=common_pb2.FAILURE,
            )
        # Assert target branch exists remotely.
        https_remote = api.sso.sso_to_https(props.remote)
        branch_revision = api.git.get_remote_branch_head(
            step_name="get target branch HEAD",
            url=https_remote,
            branch=props.target_branch,
        )
        if not branch_revision:
            return RawResult(
                summary_markdown=(
                    f"target branch {props.target_branch} does not exist"
                ),
                status=common_pb2.FAILURE,
            )

    manifest = props.manifest
    # The default checkout doesn't include the recipes repo, so it's necessary
    # to use a non-default manifest that does include the recipes repo in order
    # for recipes cherry-picks to work.
    # TODO(olivernewman): This is a hack, the better solution would be to add a
    # new Jiri manifest to integration.git that includes all existing repos as
    # well as the recipes manifest, and configure all builders that use this
    # recipe to use that manifest.
    if props.target_project == "infra/recipes":
        manifest = "infra/recipes"

    # Checkout at branch revision.
    checkout = api.checkout.fuchsia_with_options(
        build_input=build_pb2.Build.Input(
            gitiles_commit=common_pb2.GitilesCommit(
                id=branch_revision,
                project=props.project,
                host=urlparse(https_remote).hostname,
            ),
        ),
        project=props.project,
        manifest=manifest,
        remote=props.remote,
        # Skip for faster checkout.
        fetch_packages=False,
    )
    integration_root = checkout.project_path(props.project)

    with api.step.nest("resolve release version"):
        # Attempt to increment candidate version.
        release_version = api.release.get_next_candidate_version(
            ref=branch_revision,
            repo_path=integration_root,
            candidate_version_increment=props.candidate_version_increment,
        )
        if api.git.get_remote_tag(url=https_remote, tag=release_version.tag_name):
            return RawResult(
                summary_markdown=(
                    f"release version conflict: attempting increment would "
                    f"conflict with existing tag {release_version.tag_name}, "
                    f"indicating branch {props.target_branch} has already "
                    f"moved forward"
                ),
                status=common_pb2.FAILURE,
            )

    with api.step.nest("get project info"), api.context(cwd=integration_root):
        project_list = api.jiri.project(
            projects=[props.target_project],
            list_remote=True,
        ).json.output
        # Under most circumstances, the project name is unique, so we just use
        # the first and only element in these cases. However, it is possible for
        # the project name to be duplicated, so we use the `target_remote`
        # property to disambiguate.
        project_info = None
        for info in project_list:
            if api.sso.sso_to_https(info["remote"]) == props.target_remote:
                project_info = info
                break
        if not project_info:
            return RawResult(
                summary_markdown=(
                    f"invalid project input: project {props.target_project} "
                    f"with remote {props.target_remote} is not in the checkout"
                ),
                status=common_pb2.FAILURE,
            )
        # Run integrity checks: ensure `revision` is equivalent to the current
        # revision for `change_id`, and ensure that the change is approved.
        gerrit_host = api.jiri.read_manifest_element(
            project_info["manifest"], "project", props.target_project
        )["gerrithost"]
        for change_info in props.changes:
            change_details = api.gerrit.change_details(
                name="get change details",
                change_id=change_info.change_id,
                host=urlparse(gerrit_host).netloc,
                query_params=["CURRENT_REVISION"],
                test_data=api.json.test_api.output(
                    {
                        "current_revision": "foo",
                    }
                ),
            ).json.output
            change_revision = change_details["current_revision"]
            change_info.change_url = (
                f"{gerrit_host}/c/{props.target_project}/+/{change_info.change_id}"
            )
            if change_info.revision != change_revision:
                return RawResult(
                    summary_markdown=(
                        f"revision integrity check failed: specified revision "
                        f"{change_info.revision} is not equal to the resolved "
                        f"change revision {change_revision}, indicating that "
                        f"{change_info.change_url} may have been updated "
                        f"recently"
                    ),
                    status=common_pb2.FAILURE,
                )

    with api.step.nest(
        f"apply {pluralize('cherry-pick', len(props.changes))}"
    ), api.context(cwd=api.path.abs_to_path(project_info["path"])):
        commit_msgs = []
        for change_info in props.changes:
            with api.step.nest(str(change_info.change_id)):
                base_revision = api.git.get_hash()
                # Ensure revision is fetched.
                api.git.fetch("origin", refspec=change_info.revision)
                try:
                    api.git.cherry_pick(change_info.revision)
                # On failure, raise a helpful exception.
                except api.step.StepFailure:
                    return RawResult(
                        summary_markdown=f"cannot cleanly cherry-pick {change_info.change_url} revision {change_info.revision} onto base revision {base_revision} on branch {props.target_branch}",
                        status=common_pb2.FAILURE,
                    )
                commit_msgs.append(
                    api.git.get_commit_message(commit=change_info.revision)
                )
        api.git.push(
            refs=[f"HEAD:refs/heads/{props.target_branch}"],
            dryrun=props.dryrun,
        )
        new_revision = api.git.get_hash()

    # On successful cherry-pick, update project pin in integration.
    with api.context(cwd=integration_root), api.step.nest(
        "update integration"
    ) as presentation:
        if props.target_project != props.project:
            api.jiri.init()
            api.jiri.edit_manifest(
                manifest=project_info["manifest"],
                projects=[(project_info["name"], new_revision)],
            )
            update_message = COMMIT_MESSAGE.format(
                prefix=api.release._COMMIT_MESSAGE_PREFIX,
                project=props.target_project,
                revision=api.release.shorten_revision(new_revision),
                change_urls=", ".join((c.change_url for c in props.changes)),
                commit_msgs="\n\n".join(commit_msgs),
            )
            api.git.commit(
                message=update_message,
                all_tracked=True,
            )
        with api.context(infra_steps=True):
            release_revision = api.git.get_hash()
            api.git.tag(release_version.tag_name, step_name="tag release")
            # Push commit and release version.
            api.git.push(
                step_name="push release",
                refs=[
                    f"HEAD:refs/heads/{props.target_branch}",
                    f"refs/tags/{release_version.tag_name}",
                ],
                atomic=True,
                dryrun=props.dryrun,
            )
        api.release.set_output_properties(
            presentation,
            release_revision,
            release_version,
            https_remote,
        )

    if not props.dryrun and props.downstream_builders:
        api.release.lookup_builds(
            builders=props.downstream_builders,
            revision=release_revision,
            remote=https_remote,
        )


def GenTests(api):
    default_target_project = "test-project"

    branch_version = api.release.ref_to_release_version(
        "releases/0.20200101.0.1",
        nesting="resolve release version",
    )
    version_does_not_exist = api.step_data(
        "resolve release version.get remote tag", api.raw_io.stream_output_text("")
    )

    def properties(**kwargs):
        defaults = {
            "bug": "123456",
            "changes": [{"revision": "foo", "change_id": "Ia123b"}],
            "target_project": default_target_project,
            "target_remote": "https://fuchsia.googlesource.com/test-project",
            "target_branch": "releases/targetbranch",
            "project": "integration",
            "manifest": "integration",
            "remote": "sso://fuchsia/integration",
        }
        return api.properties(**{**defaults, **kwargs})

    project_info = api.jiri.read_manifest_element(
        default_target_project,
        test_output={
            "manifest": "fuchsia/manifest",
            "path": default_target_project,
            "gerrithost": "https://fuchsia-review.googlesource.com",
        },
        nesting="get project info",
    )
    merge_conflict = api.step_data(
        "apply 1 cherry-pick.Ia123b.git cherry-pick", retcode=1
    )

    yield (api.test("missing_changes", status="FAILURE") + properties(changes=None))
    yield (
        api.test("missing_target_project", status="FAILURE")
        + properties(target_project=None)
    )
    yield (
        api.test("missing_target_remote", status="FAILURE")
        + properties(target_remote=None)
    )
    yield (
        api.test("invalid_branch_input", status="FAILURE")
        + properties(
            target_branch="invalidbranch",
        )
    )

    yield (
        api.test("branch_does_not_exist", status="FAILURE")
        + properties()
        + api.step_data(
            "check inputs.get target branch HEAD",
            api.raw_io.stream_output_text(""),
        )
    )

    yield (
        api.test("release_version_conflict", status="FAILURE")
        + properties()
        + branch_version
    )

    yield (
        api.test("invalid_project_input", status="FAILURE")
        + properties()
        + branch_version
        + version_does_not_exist
        + api.step_data("get project info.jiri project", api.jiri.project([]))
    )

    yield (
        api.test("failed_integrity_check", status="FAILURE")
        + properties(
            changes=[{"revision": "notcurrent", "change_id": "Ia123b"}],
        )
        + branch_version
        + version_does_not_exist
        + project_info
    )

    yield (
        api.test("cherry_pick")
        + properties(downstream_builders=["a/b/c"])
        + branch_version
        + version_does_not_exist
        + project_info
    )

    yield (
        api.test("cherry_pick_recipes")
        + properties(
            target_project="infra/recipes",
            target_remote="https://fuchsia.googlesource.com/infra/recipes",
        )
        + branch_version
        + version_does_not_exist
        + api.jiri.read_manifest_element(
            "infra/recipes",
            test_output={
                "manifest": "fuchsia/manifest",
                "path": "infra/recipes",
                "gerrithost": "https://fuchsia-review.googlesource.com",
            },
            nesting="get project info",
        )
    )

    yield (
        api.test("merge_conflict", status="FAILURE")
        + properties()
        + branch_version
        + version_does_not_exist
        + project_info
        + merge_conflict
    )