| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Example recipe for auto-rolling.""" |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common |
| from recipe_engine.config import Dict, List |
| from recipe_engine.post_process import MustRun |
| from recipe_engine.recipe_api import Property |
| |
| from RECIPE_MODULES.fuchsia.auto_roller.api import ( |
| CQ_MESSAGE_TAGS, |
| FAILED_DRY_RUN_MESSAGE, |
| PASSED_DRY_RUN_MESSAGE, |
| ) |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| DEPS = [ |
| "fuchsia/auto_roller", |
| "fuchsia/buildbucket_util", |
| "fuchsia/gerrit", |
| "fuchsia/git", |
| "recipe_engine/json", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| ] |
| |
| PROPERTIES = { |
| "project": Property(kind=str, help="Gerrit project", default=None), |
| "remote": Property(kind=str, help="Remote repository"), |
| "commit_untracked_files": Property( |
| kind=bool, default=False, help="Whether to commit untracked files" |
| ), |
| "create_unique_id": Property( |
| kind=bool, |
| default=False, |
| help="Whether to create a Gerrit Change ID unique to this build", |
| ), |
| "dry_run": Property( |
| kind=bool, |
| default=False, |
| help="Whether to dry-run the auto-roller (CQ+1 and abandon the change)", |
| ), |
| "force_submit": Property( |
| kind=bool, |
| default=False, |
| help="Whether to force-submit the change, bypassing CQ", |
| ), |
| "no_tryjobs": Property( |
| kind=bool, |
| default=False, |
| help="Whether CQ should skip running tryjobs", |
| ), |
| "labels_to_set": Property( |
| kind=Dict(), |
| default=None, |
| help="Extra labels to set on push.", # str->int |
| ), |
| "labels_to_wait_on": Property( |
| kind=List(str), |
| default=None, |
| help="Extra labels to wait on before submitting.", |
| ), |
| "bot_commit": Property( |
| kind=bool, |
| default=False, |
| help="Set Bot-Commit", |
| ), |
| "include_tryjobs": Property( |
| kind=Dict(), default=None, help="Extra tryjobs to include" |
| ), |
| } |
| |
| |
| def RunSteps( |
| api, |
| project, |
| remote, |
| commit_untracked_files, |
| create_unique_id, |
| dry_run, |
| force_submit, |
| no_tryjobs, |
| labels_to_set, |
| labels_to_wait_on, |
| bot_commit, |
| include_tryjobs, |
| ): |
| # Check out the repo. |
| api.git.checkout(remote) |
| |
| gerrit_host = api.gerrit.host_from_remote_url(remote) |
| |
| # Do some changes to the repo. |
| # ... |
| |
| # Land the changes. |
| change = api.auto_roller.attempt_roll( |
| gerrit_host, |
| gerrit_project=project, |
| repo_dir=api.path["start_dir"].join(project), |
| commit_message="hello world!", |
| commit_untracked=commit_untracked_files, |
| force_submit=force_submit, |
| no_tryjobs=no_tryjobs, |
| cl_notify_option="ALL", |
| create_unique_id=create_unique_id, |
| dry_run=dry_run, |
| labels_to_set=labels_to_set, |
| labels_to_wait_on=labels_to_wait_on, |
| cc=["noreply@example.com"], |
| cc_on_failure=["onfailure@example.com"], |
| bot_commit=bot_commit, |
| roller_owners=["foo@example.com", "bar@example.com"], |
| include_tryjobs=include_tryjobs, |
| ) |
| if change: # For code coverage. |
| change.revision # pylint: disable=pointless-statement |
| |
| return api.auto_roller.raw_result(change) |
| |
| |
| def GenTests(api): |
| def properties(**kwargs): |
| props = { |
| "project": "integration", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "poll_interval_secs": 0.001, |
| "poll_timeout_secs": 0.1, |
| } |
| props.update(kwargs) |
| return api.properties(**props) |
| |
| yield ( |
| api.buildbucket_util.test("successful_roll", execution_timeout=60 * 60) |
| + properties(poll_timeout_secs=None) |
| + api.auto_roller.redirected_push("new-main") |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("untracked_files__with_led") |
| + properties( |
| commit_untracked_files=True, |
| **{"$recipe_engine/led": {"led_run_id": "led/user_example.com/abc123"}} |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("noop") |
| + properties() |
| + api.step_data("check for no-op commit", api.raw_io.stream_output_text("")) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "cq_failure", |
| status="failure", |
| tags=[ |
| common.StringPair(key="scheduler_job_id", value="fuchsia/trigger-123") |
| ], |
| ) |
| + properties() |
| + api.auto_roller.failure() |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("abandoned", status="failure") |
| + properties(bot_commit=True) |
| + api.auto_roller.abandoned() |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("dry_run_success", execution_timeout=5 * 60) |
| + properties( |
| # Do not collide with other concurrent dryruns. |
| create_unique_id=True, |
| dry_run=True, |
| poll_timeout_secs=None, |
| ) |
| + api.auto_roller.dry_run_success() |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("dry_run_failure", status="failure") |
| + properties(dry_run=True) |
| + api.auto_roller.dry_run_failure() |
| ) |
| |
| # If CQ removes the CQ+1 vote but doesn't post a comment with the dry run |
| # status within the next check, it should cause an infra failure. |
| yield ( |
| api.buildbucket_util.test("dry_run_no_comment", status="infra_failure") |
| + properties(dry_run=True) |
| + api.step_data( |
| "check for completion.check if done (0)", |
| api.json.output( |
| { |
| "_number": 456, |
| "current_revision": "abc", |
| "status": "NEW", |
| "labels": {"Commit-Queue": {}}, |
| "messages": [ |
| { |
| "message": "CQ is trying the patch", |
| "tag": CQ_MESSAGE_TAGS[0], |
| }, |
| ], |
| } |
| ), |
| ) |
| + api.step_data( |
| "check for completion.check if done (1)", |
| api.json.output( |
| { |
| "_number": 456, |
| "current_revision": "abc", |
| "status": "NEW", |
| "labels": {"Commit-Queue": {}}, |
| "messages": [ |
| { |
| "message": "CQ is trying the patch", |
| "tag": CQ_MESSAGE_TAGS[0], |
| }, |
| ], |
| } |
| ), |
| ) |
| ) |
| |
| # If CQ removes the CQ+1 vote but doesn't post a comment with the dry run |
| # status, it should attempt another check, and succeed if the comment is |
| # posted. |
| yield ( |
| api.buildbucket_util.test("dry_run_slow_comment") |
| + properties(dry_run=True) |
| + api.step_data( |
| "check for completion.check if done (0)", |
| api.json.output( |
| { |
| "_number": 456, |
| "current_revision": "abc", |
| "status": "NEW", |
| "labels": {"Commit-Queue": {}}, |
| "messages": [ |
| { |
| "message": "CQ is trying the patch", |
| "tag": CQ_MESSAGE_TAGS[0], |
| }, |
| ], |
| } |
| ), |
| ) |
| + api.auto_roller.dry_run_success(iteration=1) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("dry_run_two_checks") |
| + properties(dry_run=True) |
| + api.auto_roller.dry_run_incomplete(iteration=0) |
| + api.auto_roller.dry_run_success(iteration=1) |
| ) |
| |
| yield api.buildbucket_util.test("force_submit") + properties(force_submit=True) |
| |
| yield ( |
| api.buildbucket_util.test("force_submit_dry_run") |
| + properties(force_submit=True, dry_run=True) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("no_tryjobs") |
| + properties(no_tryjobs=True) |
| + api.auto_roller.success() |
| ) |
| |
| def previous_dry_run_attempt_data(success): |
| message = PASSED_DRY_RUN_MESSAGE if success else FAILED_DRY_RUN_MESSAGE |
| return api.step_data( |
| "check for identical roll", |
| api.json.output( |
| [ |
| { |
| "_number": 456, |
| "status": "ABANDONED", |
| "labels": None, |
| "messages": [ |
| {"message": message, "tag": CQ_MESSAGE_TAGS[0]}, |
| { |
| "message": "Abandoned", |
| "tag": "autogenerated:gerrit:abandon", |
| }, |
| ], |
| } |
| ] |
| ), |
| ) |
| |
| # If a previous identical CL passed a CQ dry run, exit immediately without |
| # retrying CQ. |
| yield ( |
| api.buildbucket_util.test("previous_dry_run_successful") |
| + properties(dry_run=True) |
| + previous_dry_run_attempt_data(success=True) |
| ) |
| |
| # If a previous identical CL failed a CQ dry run, exit immediately without |
| # retrying CQ and turn the build red. |
| yield ( |
| api.buildbucket_util.test("previous_dry_run_failed", status="failure") |
| + properties(dry_run=True) |
| + previous_dry_run_attempt_data(success=False) |
| ) |
| |
| # Test a failure to roll due to an auto-roller timeout. Sets the |
| # poll_timeout to be very close to the poll_interval so only one loop is |
| # made. Here, we substitute in mock data that indicates CQ is still running, |
| # but since we only try once, we will time out. |
| yield ( |
| api.buildbucket_util.test("timeout", status="failure") |
| + properties(poll_interval_secs=0.001, poll_timeout_secs=0.0015) |
| + api.auto_roller.timeout(iteration=0) |
| + api.auto_roller.timeout(iteration=1) |
| ) |
| |
| # Test a successful roll that passes after the final loop executes. We now |
| # check status after the last sleep in case the roll passed at the last |
| # second. Otherwise this test is setup as fuchsia_timeout above. |
| yield ( |
| api.buildbucket_util.test("pass_last_second") |
| + properties() |
| + api.auto_roller.timeout(iteration=0) |
| + api.auto_roller.success(iteration=1) |
| ) |
| |
| # Test a successful roll with integral arguments to poll_*_secs. This tests |
| # for any regression in supporting integral values for polling-related |
| # properties. |
| yield ( |
| api.buildbucket_util.test("integral_poll_secs") |
| + properties(poll_interval_secs=1, poll_timeout_secs=1) |
| + api.auto_roller.success() |
| ) |
| |
| # If we find a previous roll with the same diff that has been abandoned, we |
| # should restore it and retry CQ. |
| yield ( |
| api.buildbucket_util.test("existing_roll") |
| + properties() |
| + api.step_data( |
| "check for identical roll", |
| api.json.output( |
| [ |
| { |
| "_number": 123, |
| "change_id": "Iabc", |
| "status": "ABANDONED", |
| "current_revision": "abc", |
| "revisions": { |
| "abc": { |
| "commit": {"message": "[roll] Previous commit message"} |
| } |
| }, |
| } |
| ] |
| ), |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| # If we find a previous roll CL with the same diff that's merged, we should |
| # calculate a new Change-Id to avoid a collision. |
| yield ( |
| api.buildbucket_util.test("existing_roll_merged") |
| + properties() |
| + api.step_data( |
| "check for identical roll", |
| api.json.output( |
| [ |
| { |
| "_number": 123, |
| "change_id": "Iabc", |
| "status": "MERGED", |
| "current_revision": "abc", |
| "revisions": { |
| "abc": { |
| "commit": {"message": "[roll] Previous commit message"} |
| } |
| }, |
| } |
| ] |
| ), |
| ) |
| # We should also check to make the recalculated Change-Id doesn't |
| # collide with another change. |
| + api.post_process(MustRun, "check for identical roll (2)") |
| + api.auto_roller.success() |
| ) |
| |
| # If we detect a previous identical roll CL that is still open, we should |
| # attempt to re-CQ it. |
| yield ( |
| api.buildbucket_util.test("existing_roll_not_abandoned") |
| + properties() |
| + api.step_data( |
| "check for identical roll", |
| api.json.output( |
| [ |
| { |
| "_number": 123, |
| "change_id": "Iabc", |
| "status": "OPEN", |
| "current_revision": "abc", |
| "revisions": { |
| "abc": { |
| "commit": {"message": "[roll] Previous commit message"} |
| } |
| }, |
| } |
| ] |
| ), |
| ) |
| + api.auto_roller.success() |
| ) |
| |
| # Set extra labels on push, and wait for extra labels at end. |
| yield ( |
| api.buildbucket_util.test("test_extra_labels") |
| + properties( |
| labels_to_set={"Trigger": 1}, |
| labels_to_wait_on=("Verified",), |
| ) |
| + api.auto_roller.success( |
| labels={"Verified": {"approved": {}}, "Commit-Queue": {}} |
| ) |
| ) |
| |
| # Rejected extra label. |
| yield ( |
| api.buildbucket_util.test("test_extra_labels_rejected", status="failure") |
| + properties( |
| labels_to_set={"Trigger": 1}, |
| labels_to_wait_on=("Verified",), |
| ) |
| + api.auto_roller.failure( |
| labels={"Verified": {"rejected": {}}, "Commit-Queue": {}} |
| ) |
| ) |
| |
| # No Commit-Queue label on target repository. |
| yield ( |
| api.buildbucket_util.test("no_commit_queue") |
| + properties( |
| labels_to_set={"Commit-Queue": 0}, |
| labels_to_wait_on=("Verified",), |
| ) |
| # Not looking for failure but without CQ failure and success look the |
| # same. |
| + api.auto_roller.timeout(labels={"Verified": {}}, iteration=0) |
| + api.auto_roller.failure(labels={"Verified": {"approved": {}}}, iteration=1) |
| ) |
| |
| # Set extra tryjobs to include in the dry run roll. |
| yield ( |
| api.buildbucket_util.test("include_tryjobs") |
| + properties( |
| dry_run=True, |
| include_tryjobs={"luci.fuchsia.try": ["fuchsia-coverage"]}, |
| ) |
| + api.auto_roller.dry_run_success() |
| ) |
