recipe_modules/gerrit_auto_submit/api.py

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import datetime

from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2
from PB.recipe_engine import result
from recipe_engine import recipe_api

# Timeout for Gerrit API operations.
GERRIT_TIMEOUT = datetime.timedelta(seconds=120)

CQ_MESSAGE_TAGS = (
    "autogenerated:cv",
    # TODO(olivernewman): Delete these entries one LUCI CV is used everywhere
    # and we no longer need to support using the Commit Queue service.
    "autogenerated:cq:dry-run",
    "autogenerated:cq:full-run",
)


class GerritAutoSubmitApi(recipe_api.RecipeApi):
    def __init__(self, props, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._gerrit_host = props.gerrit_host or "fuchsia-review.googlesource.com"
        self._auto_submit_label = props.auto_submit_label or "Fuchsia-Auto-Submit"
        self._tree_status_host = props.tree_status_host
        self._max_attempts = props.max_attempts or 4

    def _get_change_url(self, change):
        return "https://%s/c/%s/+/%s" % (
            self._gerrit_host,
            change["project"],
            change["_number"],
        )

    def _is_auto_submit_service_account(self, email):
        try:
            # The current implementation of get_email() raises
            # NotImplementedError but this lets us switch when it is
            # implemented.
            return email == self.m.service_account.default().get_email()
        except NotImplementedError:
            # Most service accounts doing auto-submit have "auto-submit" in
            # their service account email.
            # TODO(crbug.com/1275620) Remove this except block.
            return "auto-submit" in email

    def get_eligible_changes(self):
        raw_query = " ".join(
            [
                "is:submittable",
                "is:open",
                "-is:wip",
                "-(label:Commit-Queue+1 OR label:Commit-Queue+2)",
                "label:{}+1",
                # Ignore changes that haven't been updated in a while to keep
                # response sizes small. We look back at least a few days to ensure
                # that we can recover missed CLs in case the auto-submit builder
                # breaks for a while.
                "-age:10d",
                # Ignore changes with unresolved comments so that changes don't
                # get submitted until all reviewer feedback has been addressed.
                "-has:unresolved",
            ]
        ).format(self._auto_submit_label)
        changes = self.m.gerrit.change_query(
            name="get changes for %s" % self._gerrit_host,
            query_string=raw_query,
            query_params=["CURRENT_REVISION"],
            host=self._gerrit_host,
            max_attempts=3,
            timeout=GERRIT_TIMEOUT,
        ).json.output

        # API returns None if there is no search results instead of [].
        if not changes:
            return []

        eligible_changes = []
        for change in changes:
            change_id = self.m.url.unquote(change["id"])

            with self.m.step.nest(
                "get messages for %s" % change["_number"]
            ) as presentation:
                presentation.links["change"] = self._get_change_url(change)
                messages = self.m.gerrit.change_details(
                    "get details",
                    change_id,
                    host=self._gerrit_host,
                    max_attempts=3,
                    timeout=GERRIT_TIMEOUT,
                    test_data=self.m.json.test_api.output({"messages": []}),
                ).json.output["messages"]

                # Some prod accounts don't have emails so we need to use .get().
                updates = [
                    msg["author"].get("email", "")
                    for msg in messages
                    if msg.get("tag") not in CQ_MESSAGE_TAGS
                ]

                # Stop retrying if the last `max_attempts` comments are all
                # from the auto-submit service account.
                if len(updates) >= self._max_attempts and all(
                    self._is_auto_submit_service_account(email)
                    for email in updates[-self._max_attempts :]
                ):
                    continue

            with self.m.step.nest(
                "get details for %s" % change["_number"]
            ) as presentation:
                presentation.links["change"] = self._get_change_url(change)
                other_changes_info = self.m.gerrit.changes_submitted_together(
                    "find dependent changes",
                    change_id,
                    query_params=["NON_VISIBLE_CHANGES"],
                    host=self._gerrit_host,
                    max_attempts=3,
                    timeout=GERRIT_TIMEOUT,
                ).json.output

                if (
                    len(other_changes_info["changes"]) > 1
                    or other_changes_info.get("non_visible_changes", 0) != 0
                ):
                    continue
                try:
                    change_mergeability = self.m.gerrit.get_mergeable(
                        "get mergeable",
                        change_id,
                        host=self._gerrit_host,
                        max_attempts=3,
                        timeout=GERRIT_TIMEOUT,
                    ).json.output
                except self.m.step.StepFailure:  # pragma: no cover
                    # If this query fails we shouldn't fail the entire
                    # auto-submit run, and instead skip this change and continue
                    # trying to process other changes.
                    continue
                if not change_mergeability["mergeable"]:
                    # Best-effort attempt at a trivial rebase. If it fails, that
                    # means there's a true merge conflict that must be resolved
                    # manually by the CL owner, so we won't try to CQ the change.
                    # TODO(olivernewman) `get_eligible_changes()` shouldn't modify
                    # state; refactor to do this in a post-process filtering step.
                    if self.m.gerrit.rebase(
                        "rebase",
                        change_id,
                        host=self._gerrit_host,
                        ok_ret="any",
                        timeout=GERRIT_TIMEOUT,
                    ).retcode:
                        continue
                eligible_changes.append(change)
        return eligible_changes

    def set_commit_queue(self, change):
        step = self.m.gerrit.set_review(
            name="%s" % change["_number"],
            change_id=self.m.url.unquote(change["id"]),
            labels={"Commit-Queue": 2},
            host=self._gerrit_host,
            notify="NONE",
            ok_ret="any",
            timeout=GERRIT_TIMEOUT,
        )
        step.presentation.links["change"] = self._get_change_url(change)
        return step.retcode == 0

    def raw_result(self, changes, dry_run):
        if changes:
            contents = [
                "Submitted the following CLs{}:".format(" (dry run)" if dry_run else "")
            ]
            contents.extend(self._get_change_url(change) for change in changes)
        else:
            contents = ["No CLs to submit."]

        return result.RawResult(
            summary_markdown="  \n".join(contents),
            status=common_pb2.SUCCESS,
        )

    def filter_state(self, now, state):
        delete = []
        for k, v in state.items():
            if now - v["last_action_time_secs"] > 24 * 60 * 60:
                delete.append(k)
        for k in delete:
            del state[k]
        return state

    def can_try_submit(self, now, change_state, tree_status):
        # Never seen this change before.
        if not change_state:
            return True
        elif tree_status and not tree_status.open:
            # Don't retry CQ if the tree is closed, to avoid overwhelming CQ during
            # widespread breakages.
            return False

        # Check if we attempted to submit too recently.
        if (
            now - change_state["last_action_time_secs"]
            > 2 ** change_state["attempts"] * 30 * 60
        ):
            if change_state["attempts"] < self._max_attempts:
                return True
        return False

    def submit(self, now, state, changes, tree_status):
        submitted_changes = []
        for change in changes:
            current_revision = change["current_revision"]
            change_state = state.get(current_revision, {})
            if self.can_try_submit(now, change_state, tree_status):
                if self.set_commit_queue(change):
                    change_state["attempts"] = change_state.get("attempts", 0) + 1
                    change_state["last_action_time_secs"] = now
                    state[current_revision] = change_state
                    submitted_changes.append(change)
        return submitted_changes

    # TODO(nmulcahey): Add a Change class to the GerritApi to simplify this, and
    # other logic in recipes.
    def __call__(self, dry_run):
        tree_status = None
        if self._tree_status_host:
            tree_status = self.m.tree_status.get(self._tree_status_host)
        state = self.m.builder_state.fetch_previous_state()
        changes = None
        try:
            with self.m.step.nest("get eligible") as presentation:
                changes = self.get_eligible_changes()
                if not changes:
                    presentation.step_text = "\nno eligible changes."

            if changes:
                with self.m.step.nest("cq") as presentation:
                    if dry_run:
                        for change in [
                            self._get_change_url(change) for change in changes
                        ]:
                            presentation.links[change] = change
                    else:
                        now = int(self.m.time.time())
                        # Drop all state for changes that are no longer eligible
                        # (e.g. because they've been submitted)
                        state = self.filter_state(now, state)
                        # Attempt to submit all the changes we found.
                        changes = self.submit(now, state, changes, tree_status)

        finally:
            self.m.builder_state.save(state)

        return self.raw_result(changes, dry_run)