recipe_modules/auto_roller/tests/full.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Example recipe for auto-rolling."""

from recipe_engine.post_process import MustRun

from RECIPE_MODULES.fuchsia.auto_roller.api import (
    CQ_SERVICE_ACCOUNT_DOMAIN,
    FAILED_DRY_RUN_MESSAGES,
    PASSED_DRY_RUN_MESSAGES,
)

from PB.go.chromium.org.luci.buildbucket.proto import common
from PB.recipe_modules.fuchsia.auto_roller.tests.full import InputProperties

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/git_checkout",
    "recipe_engine/json",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/time",
]

PROPERTIES = InputProperties


def RunSteps(api, props):
    checkout_dir, _ = api.git_checkout(props.remote)

    # Do some changes to the repo.
    # ...

    # Generate a commit message (for CIPD package rolls only).
    api.auto_roller.generate_package_roll_message(
        ["foo", "bar"],
        "version:2",
        old_version="version:1",
        dry_run=props.roll_options.dry_run,
    )

    # Land the changes.
    change = api.auto_roller.attempt_roll(
        props.roll_options,
        repo_dir=checkout_dir,
        commit_message="hello world!",
    )
    if change:  # For code coverage.
        change.revision  # pylint: disable=pointless-statement

    return api.auto_roller.raw_result(change)


def GenTests(api):
    def test(*args, **kwargs):
        kwargs.setdefault("execution_timeout", 100)
        return (
            api.buildbucket_util.test(*args, **kwargs)
            + api.time.seed(0)
            + api.time.step(1)
        )

    def properties(
        create_unique_change_id=False,
        dry_run=False,
        commit_untracked=False,
        no_tryjobs=False,
        force_submit=False,
        include_tryjobs=None,
        labels_to_set=None,
        labels_to_wait_on=None,
        bot_commit=False,
        add_gitwatcher_ignore=False,
        reviewer_emails=None,
        permit_recommended=False,
        wait_for_merge=False,
        cq_retries=0,
        **kwargs,
    ):
        remote = "https://fuchsia.googlesource.com/integration"
        props = {
            "remote": remote,
            "roll_options": api.auto_roller.Options(
                remote=remote,
                bot_commit=bot_commit,
                cc_emails=["noreply@example.com"],
                cc_on_failure_emails=["onfailure@example.com"],
                cl_notify_option="ALL",
                commit_untracked=commit_untracked,
                create_unique_change_id=create_unique_change_id,
                dry_run=dry_run,
                force_submit=force_submit,
                include_tryjobs=include_tryjobs,
                labels_to_set=labels_to_set,
                labels_to_wait_on=labels_to_wait_on,
                no_tryjobs=no_tryjobs,
                roller_owners=["foo@example.com", "bar@example.com"],
                add_gitwatcher_ignore=add_gitwatcher_ignore,
                reviewer_emails=reviewer_emails,
                poll_interval_secs=10,
                permit_recommended=permit_recommended,
                wait_for_merge=wait_for_merge,
                push_options=["opt1"],
                cq_retries=cq_retries,
            ),
        }
        props.update(kwargs)
        return api.properties(**props)

    yield (
        test("keep_cl_for_manual_review", status="SUCCESS")
        + properties(
            dry_run=True,
            reviewer_emails=["reviewer@example.com"],
        )
    )

    yield (
        test("successful_roll", execution_timeout=60 * 60)
        + properties(add_gitwatcher_ignore=True)
        + api.step_data(
            "git push",
            stdout=api.raw_io.output_text(
                "remote: Update redirected to refs/for/refs/heads/new-main"
                "%l=Bot-Commit+1,l=Commit-Queue+2."
            ),
        )
        + api.auto_roller.success()
    )

    yield (
        test("retry_push")
        + properties()
        + api.step_data("git push", retcode=1)
        # If a push step fails because the change was already pushed, it should
        # be considered a success.
        + api.step_data(
            "git push (2)",
            stderr=api.raw_io.output_text("push failed (no new changes)"),
            retcode=1,
        )
        + api.auto_roller.success()
    )

    yield (
        test("untracked_files__with_led")
        + properties(
            commit_untracked=True,
            **{"$recipe_engine/led": {"led_run_id": "led/user_example.com/abc123"}},
        )
        + api.auto_roller.success()
    )

    yield (
        test("noop")
        + properties()
        + api.step_data("check for no-op commit", api.raw_io.stream_output_text(""))
    )

    yield (
        test(
            "cq_failure",
            status="FAILURE",
            tags=[
                common.StringPair(key="scheduler_job_id", value="fuchsia/trigger-123")
            ],
        )
        + properties(cq_retries=1)
        + api.auto_roller.failure()
        + api.auto_roller.failure(name="check for completion (2).check if done ({})")
    )

    yield (
        test("abandoned", status="FAILURE")
        + properties(bot_commit=True)
        + api.auto_roller.abandoned()
    )

    yield (
        test("dry_run_success", execution_timeout=5 * 60)
        + properties(
            # Do not collide with other concurrent dryruns.
            create_unique_change_id=True,
            dry_run=True,
            cq_retries=2,
        )
        + api.auto_roller.dry_run_failure()
        + api.auto_roller.dry_run_success(
            name="check for completion (2).check if done ({})"
        )
    )

    yield (
        test("dry_run_failure", status="FAILURE")
        + properties(dry_run=True)
        + api.auto_roller.dry_run_failure()
    )

    # If CQ removes the CQ+1 vote but doesn't post a comment with the dry run
    # status within the next check, it should cause an infra failure. Also,
    # ignore any messages that come before "CQ is trying the patch".
    yield (
        test("dry_run_no_comment", status="INFRA_FAILURE")
        + properties(dry_run=True)
        + api.step_data(
            "check for completion.check if done (0)",
            api.json.output(
                {
                    "_number": 456,
                    "current_revision": "abc",
                    "status": "NEW",
                    "labels": {"Commit-Queue": {}},
                    "messages": [
                        {
                            "message": "CQ is trying the patch",
                            "real_author": {
                                "email": f"foo@{CQ_SERVICE_ACCOUNT_DOMAIN}",
                            },
                        },
                    ],
                }
            ),
        )
        + api.step_data(
            "check for completion.check if done (1)",
            api.json.output(
                {
                    "_number": 456,
                    "current_revision": "abc",
                    "status": "NEW",
                    "labels": {"Commit-Queue": {}},
                    "messages": [
                        {
                            "message": "This CL has passed the run",
                            "real_author": {
                                "email": f"foo@{CQ_SERVICE_ACCOUNT_DOMAIN}",
                            },
                        },
                        {
                            "message": "CQ is trying the patch",
                            "real_author": {
                                "email": f"foo@{CQ_SERVICE_ACCOUNT_DOMAIN}",
                            },
                        },
                    ],
                }
            ),
        )
    )

    # If CQ removes the CQ+1 vote but doesn't post a comment with the dry run
    # status, it should attempt another check, and succeed if the comment is
    # posted.
    yield (
        test("dry_run_slow_comment")
        + properties(dry_run=True)
        + api.step_data(
            "check for completion.check if done (0)",
            api.json.output(
                {
                    "_number": 456,
                    "current_revision": "abc",
                    "status": "NEW",
                    "labels": {"Commit-Queue": {}},
                    "messages": [
                        {
                            "message": "CQ is trying the patch",
                            "real_author": {
                                "email": f"foo@{CQ_SERVICE_ACCOUNT_DOMAIN}",
                            },
                        },
                    ],
                }
            ),
        )
        + api.auto_roller.dry_run_success(iteration=1)
    )

    yield (
        test("dry_run_two_checks")
        + properties(dry_run=True)
        + api.auto_roller.dry_run_incomplete(iteration=0)
        + api.auto_roller.dry_run_success(iteration=1)
    )

    yield test("force_submit") + properties(force_submit=True)

    yield (test("force_submit_dry_run") + properties(force_submit=True, dry_run=True))

    yield (test("no_tryjobs") + properties(no_tryjobs=True) + api.auto_roller.success())

    def previous_dry_run_attempt_data(success):
        message = (
            PASSED_DRY_RUN_MESSAGES[-1] if success else FAILED_DRY_RUN_MESSAGES[-1]
        )
        return api.step_data(
            "check for identical roll",
            api.json.output(
                [
                    {
                        "_number": 456,
                        "status": "ABANDONED",
                        "labels": None,
                        "messages": [
                            {
                                "message": message,
                                "real_author": {
                                    "email": f"foo@{CQ_SERVICE_ACCOUNT_DOMAIN}",
                                },
                            },
                            {
                                "message": "Abandoned",
                                "tag": "autogenerated:gerrit:abandon",
                            },
                        ],
                    }
                ]
            ),
        )

    # If a previous identical CL passed a CQ dry run, exit immediately without
    # retrying CQ.
    yield (
        test("previous_dry_run_successful")
        + properties(dry_run=True)
        + previous_dry_run_attempt_data(success=True)
    )

    # If a previous identical CL failed a CQ dry run, exit immediately without
    # retrying CQ and turn the build red.
    yield (
        test("previous_dry_run_failed", status="FAILURE")
        + properties(dry_run=True)
        + previous_dry_run_attempt_data(success=False)
    )

    # Test a failure to roll due to an auto-roller timeout. Sets the
    # execution_timeout to be very close to the poll_interval so only one loop
    # is made. Here, we substitute in mock data that indicates CQ is still
    # running, but since we only try once, we will time out.
    yield (
        test("timeout", status="FAILURE", execution_timeout=16)
        + properties()
        + api.auto_roller.timeout(iteration=0)
        + api.auto_roller.timeout(iteration=1)
    )

    # Test a successful roll that passes after the final loop executes. We now
    # check status after the last sleep in case the roll passed at the last
    # second. Otherwise this test is setup as fuchsia_timeout above.
    yield (
        test("pass_last_second")
        + properties()
        + api.auto_roller.timeout(iteration=0)
        + api.auto_roller.success(iteration=1)
    )

    # If we find a previous roll with the same diff that has been abandoned, we
    # should restore it and retry CQ.
    yield (
        test("existing_roll")
        + properties()
        + api.step_data(
            "check for identical roll",
            api.json.output(
                [
                    {
                        "_number": 123,
                        "change_id": "Iabc",
                        "status": "ABANDONED",
                        "current_revision": "abc",
                        "revisions": {
                            "abc": {
                                "commit": {"message": "[roll] Previous commit message"}
                            }
                        },
                    }
                ]
            ),
        )
        + api.auto_roller.success()
    )

    # If we find a previous roll CL with the same diff that's merged, we should
    # calculate a new Change-Id to avoid a collision.
    yield (
        test("existing_roll_merged")
        + properties()
        + api.step_data(
            "check for identical roll",
            api.json.output(
                [
                    {
                        "_number": 123,
                        "change_id": "Iabc",
                        "status": "MERGED",
                        "current_revision": "abc",
                        "revisions": {
                            "abc": {
                                "commit": {"message": "[roll] Previous commit message"}
                            }
                        },
                    }
                ]
            ),
        )
        # We should also check to make the recalculated Change-Id doesn't
        # collide with another change.
        + api.post_process(MustRun, "check for identical roll (2)")
        + api.auto_roller.success()
    )

    # If we detect a previous identical roll CL that is still open, we should
    # attempt to re-CQ it.
    yield (
        test("existing_roll_not_abandoned")
        + properties()
        + api.step_data(
            "check for identical roll",
            api.json.output(
                [
                    {
                        "_number": 123,
                        "change_id": "Iabc",
                        "status": "OPEN",
                        "current_revision": "abc",
                        "revisions": {
                            "abc": {
                                "commit": {"message": "[roll] Previous commit message"}
                            }
                        },
                    }
                ]
            ),
        )
        + api.auto_roller.success()
    )

    # Set extra labels on push, and wait for extra labels at end.
    yield (
        test("test_extra_labels")
        + properties(
            labels_to_set={"Trigger": 1},
            labels_to_wait_on=["Verified"],
        )
        + api.auto_roller.dry_run_success(labels={"Verified": {"approved": {}}})
    )

    # Recommended extra label, permit_recommended = False.
    yield (
        test("test_extra_labels_recommended", status="FAILURE")
        + properties(
            labels_to_wait_on=["Verified"],
        )
        + api.auto_roller.failure(labels={"Verified": {"recommended": {}}})
    )

    # Recommended extra label, permit_recommended = True.
    yield (
        test("test_extra_labels_permit_recommended")
        + properties(
            labels_to_wait_on=["Verified"],
            permit_recommended=True,
        )
        + api.auto_roller.dry_run_success(labels={"Verified": {"recommended": {}}})
    )

    # Disliked extra label.
    yield (
        test("test_extra_labels_disliked", status="FAILURE")
        + properties(
            labels_to_wait_on=["Verified"],
        )
        + api.auto_roller.failure(labels={"Verified": {"disliked": {}}})
    )

    # Rejected extra label.
    yield (
        test("test_extra_labels_rejected", status="FAILURE")
        + properties(
            labels_to_wait_on=["Verified"],
        )
        + api.auto_roller.failure(labels={"Verified": {"rejected": {}}})
    )

    # Verified label not found
    yield (
        test("test_extra_labels_not_found", status="FAILURE")
        + properties(
            labels_to_wait_on=["Verified"],
        )
        + api.auto_roller.failure(labels={})
    )

    # No Commit-Queue label on target repository.
    yield (
        test("no_commit_queue")
        + properties(
            labels_to_set={"Commit-Queue": 0},
            labels_to_wait_on=["Verified"],
        )
        # Not looking for failure but without CQ failure and success look the
        # same.
        + api.auto_roller.timeout(labels={"Verified": {}}, iteration=0)
        + api.auto_roller.failure(labels={"Verified": {"approved": {}}}, iteration=1)
    )

    # Set extra tryjobs to include in the dry run roll.
    yield (
        test("include_tryjobs")
        + properties(
            dry_run=True,
            include_tryjobs={"luci.fuchsia.try": ["fuchsia-coverage"]},
        )
        + api.auto_roller.dry_run_success()
    )

    # Set wait_for_merge to wait for CL with "MERGED" status, ignoring other Gerrit labels.
    yield (
        test("wait_for_merge")
        + properties(
            labels_to_set={"Commit-Queue": 0},
            wait_for_merge=True,
        )
        + api.step_data(
            "check for completion.check if done (0)",
            api.json.output(
                {
                    "_number": 456,
                    "current_revision": "abc",
                    "status": "NEW",
                }
            ),
        )
        + api.step_data(
            "check for completion.check if done (1)",
            api.json.output(
                {
                    "_number": 456,
                    "current_revision": "abc",
                    "status": "MERGED",
                }
            ),
        )
        + api.auto_roller.success(iteration=1)
    )