| #!/usr/bin/env fuchsia-vendored-python |
| # |
| # Copyright 2020 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Manage submitting a large number of CLs.""" |
| |
| import argparse |
| import enum |
| import re |
| import sys |
| from typing import Any, Callable, Dict, List, Optional, Tuple |
| |
| import ansi |
| import gerrit_util |
| import util |
| |
| |
| class ChangeStatus(enum.Enum): |
| UNKNOWN = 1 |
| MISSING_VOTES = 2 |
| UNRESOLVED_COMMENTS = 3 |
| READY = 4 |
| TESTING = 5 |
| SUBMITTING = 6 |
| MERGED = 7 |
| |
| def description(self) -> str: |
| # Return a brief description of a CL's state. |
| return { |
| ChangeStatus.UNKNOWN: "unknown", |
| ChangeStatus.MISSING_VOTES: "missing votes", |
| ChangeStatus.UNRESOLVED_COMMENTS: "comments", |
| ChangeStatus.READY: "ready", |
| ChangeStatus.TESTING: "testing", |
| ChangeStatus.SUBMITTING: "submitting", |
| ChangeStatus.MERGED: "merged", |
| }[self] |
| |
| def color(self) -> Callable[[str], str]: |
| """Return a color function from the ansi module for this status.""" |
| return { |
| ChangeStatus.UNKNOWN: ansi.red, |
| ChangeStatus.MISSING_VOTES: ansi.red, |
| ChangeStatus.UNRESOLVED_COMMENTS: ansi.yellow, |
| ChangeStatus.READY: ansi.green, |
| ChangeStatus.TESTING: ansi.green, |
| ChangeStatus.SUBMITTING: ansi.bright_green, |
| ChangeStatus.MERGED: ansi.gray, |
| }[self] |
| |
| def submit_error_description(self) -> Optional[str]: |
| # Return an explanation of why this CL cannot be submitted. |
| return { |
| ChangeStatus.UNKNOWN: "unknown error", |
| ChangeStatus.MISSING_VOTES: "CL is missing votes", |
| ChangeStatus.UNRESOLVED_COMMENTS: "CL has unresolved comments", |
| }.get(self) |
| |
| |
| class Change: |
| """A Gerrit Changelist.""" |
| |
| def __init__(self, change_id: str, json: Any): |
| self.change_id: str = change_id |
| self.subject: str = json.get("subject", "<unknown>") |
| self.id: int = int(json.get("_number", "-1")) |
| self.json = json |
| self.status_string: str = json.get("status", "UNKNOWN") |
| |
| @classmethod |
| def from_json(cls, json: Any) -> "Change": |
| return Change(json["change_id"], json=json) |
| |
| def labels(self) -> Dict[str, Any]: |
| labels = self.json.get("labels", {}) |
| assert isinstance(labels, Dict) |
| return labels |
| |
| def cq_votes(self) -> int: |
| cq_labels = self.labels().get("Commit-Queue", {}) |
| if cq_labels.get("approved") is not None: |
| return 2 |
| if cq_labels.get("recommended") is not None: |
| return 1 |
| return 0 |
| |
| def has_unresolved_comments(self) -> bool: |
| unresolved_comments: int = self.json.get("unresolved_comment_count", 0) |
| return unresolved_comments > 0 |
| |
| def submittable(self) -> bool: |
| is_submittable: bool = self.json.get("submittable", False) |
| return is_submittable |
| |
| @property |
| def status(self) -> ChangeStatus: |
| """Return a CL's status.""" |
| # We expect a CL to either be 'NEW' or 'MERGED'. |
| if self.status_string == "MERGED": |
| return ChangeStatus.MERGED |
| if self.status_string != "NEW": |
| return ChangeStatus.UNKNOWN |
| |
| if not self.submittable(): |
| return ChangeStatus.MISSING_VOTES |
| if self.has_unresolved_comments(): |
| return ChangeStatus.UNRESOLVED_COMMENTS |
| if self.cq_votes() == 1: |
| return ChangeStatus.TESTING |
| if self.cq_votes() == 2: |
| return ChangeStatus.SUBMITTING |
| return ChangeStatus.READY |
| |
| |
| class GerritServer: |
| """A connection to a Gerrit server.""" |
| |
| DEFAULT_PARAMS = [ |
| "LABELS", |
| "ALL_REVISIONS", |
| "SKIP_DIFFSTAT", |
| "SUBMITTABLE", |
| "CHECK", |
| ] |
| |
| def __init__(self, host: str): |
| """Create a connection to the server.""" |
| self.host = host |
| |
| def query(self, query: List[Tuple[str, str]]) -> List[Change]: |
| """Query a Gerrit server for changes matching query terms. |
| |
| Args: |
| query: A list of key:value pairs for search parameters, as documented |
| here (e.g. ('is', 'owner') for a parameter 'is:owner'): |
| https://gerrit-review.googlesource.com/Documentation/user-search.html#search-operators |
| |
| Returns: |
| JSON |
| """ |
| return [ |
| Change.from_json(x) |
| for x in gerrit_util.QueryChanges( |
| self.host, query, o_params=GerritServer.DEFAULT_PARAMS |
| ) |
| ] |
| |
| def fetch_change(self, id: str) -> Optional[Change]: |
| """Fetch information about the CL with the given Gerrit ID.""" |
| result = self.query([("change", id)]) |
| if len(result) == 1: |
| return result[0] |
| if len(result) > 1: |
| raise Exception("More than one CL had the specified change ID.") |
| return None |
| |
| def set_cq_state(self, id: str, state: int) -> None: |
| """Update the 'Commit-Queue' label to the given vote.""" |
| gerrit_util.SetReview( |
| self.host, id, labels={"Commit-Queue": state}, notify=False |
| ) |
| |
| def get_change_dependencies(self, id: str) -> List[str]: |
| """Get a list of IDs that are dependencies of the given change.""" |
| # Get related change sets. This includes both children and parent changes. |
| json: Any = gerrit_util.GetRelatedChanges(self.host, id) |
| |
| # Changes are given in topological order, from newest to oldest. |
| # |
| # Search through the changes until we see ourselves. Any later changes are dependencies. |
| dependencies: List[str] = [] |
| seen_self = False |
| for change in json.get("changes", []): |
| # Ignore children of ourselves. |
| change_id: str = change["change_id"] |
| if change_id == id: |
| seen_self = True |
| if not seen_self: |
| continue |
| dependencies.append(change_id) |
| if not seen_self: |
| return [id] |
| return dependencies |
| |
| |
| def should_submit( |
| cl: Change, abort_on_unresolved_comments: bool = True |
| ) -> bool: |
| if not cl.submittable(): |
| return False |
| if abort_on_unresolved_comments and cl.has_unresolved_comments(): |
| return False |
| return True |
| |
| |
| def shorten(s: str, max_len: int = 60) -> str: |
| """Truncate a long string, appending '...' if a change is made.""" |
| if len(s) < max_len: |
| return s |
| return s[: max_len - 3] + "..." |
| |
| |
| def print_changes(results: List[Change]) -> None: |
| """Display a list of results in a table.""" |
| print() |
| print("%20s %-10s %-65s" % ("Status", "CL Number", "Subject")) |
| print("%20s %-10s %-65s" % ("――――――", "――――――――――", "―――――――")) |
| for result in results: |
| status = result.status |
| print( |
| "%s %s %s" |
| % ( |
| status.color()("%20s" % status.description()), |
| "%-10s" % result.id, |
| "%-65s" % shorten(result.subject, 65), |
| ) |
| ) |
| print() |
| |
| |
| class SubmitError(Exception): |
| def __init__(self, message: str): |
| self.message: str = message |
| super().__init__(message) |
| |
| |
| def ensure_changes_submittable( |
| changes: List[Change], |
| abort_on_unresolved_comments: bool = True, |
| ) -> None: |
| """Ensure that the given list of changes are submittable.""" |
| for cl in changes: |
| if cl.status != ChangeStatus.MERGED and not should_submit( |
| cl, abort_on_unresolved_comments |
| ): |
| raise SubmitError( |
| "CL %d can not be submitted: %s" |
| % ( |
| cl.id, |
| cl.status.submit_error_description() or "unknown error", |
| ) |
| ) |
| |
| |
| def submit_changes( |
| clock: util.Clock, |
| server: GerritServer, |
| changes: List[Change], |
| num_retries: int = 0, |
| ) -> bool: |
| # Strip out merged changes. |
| changes = [cl for cl in changes if cl.status != ChangeStatus.MERGED] |
| |
| # For any CL that doesn't have a CQ+1, run it now to speed things up. |
| # As long as the CL isn't changed in the mean-time, it won't be tested |
| # again when we finally get around to +2'ing it. |
| # |
| # We ignore the first one, because we are just about to +2 it anyway. |
| for cl in changes[1:]: |
| if cl.cq_votes() == 0: |
| print("Setting CQ state of CL %d to dry-run." % cl.id) |
| server.set_cq_state(cl.change_id, 1) |
| |
| # Submit the changes in order. |
| for cl in changes: |
| backoff = util.ExponentialBackoff(clock) |
| max_attempts = num_retries + 1 |
| num_attempts = 0 |
| print() |
| print("Submitting CL %d: %s" % (cl.id, cl.subject)) |
| |
| # If the CL was already CQ+2'ed when we started, treat this as one of our attempts. |
| if cl.cq_votes() == 2: |
| num_attempts += 1 |
| |
| while True: |
| # Fetch the latest information about the CL. |
| current_cl = server.fetch_change(cl.change_id) |
| |
| # Check it still exists. |
| if current_cl is None: |
| raise SubmitError("CL %s could not be found." % cl.id) |
| |
| # If it is merged, we are done. |
| if current_cl.status == ChangeStatus.MERGED: |
| break |
| |
| # If it is not in CQ, add it to CQ. |
| if current_cl.cq_votes() < 2: |
| if num_attempts == 0: |
| print(ansi.gray(" Adding to CQ.")) |
| elif num_attempts < max_attempts: |
| print(ansi.yellow(" CL failed in CQ. Retrying...")) |
| else: |
| print(ansi.red(" CL failed in CQ. Aborting.")) |
| return False |
| num_attempts += 1 |
| server.set_cq_state(cl.change_id, 2) |
| |
| # wait. |
| backoff.wait() |
| print(ansi.gray(" Polling...")) |
| |
| # Did we fail? |
| print(ansi.green(" Submitted!")) |
| |
| return True |
| |
| |
| def parse_args() -> Any: |
| description = r""" |
| Submit a chain of CLs, specified by giving the CL number of the end of the |
| chain. The command will poll indefinitely until the chain is submitted or an |
| error is detected. |
| |
| The tool can be safely cancelled at any time. When restarted, it will resume |
| where it left off. |
| |
| For example, given a chain of three CLs: |
| |
| 101: Start hacking on 'foo'. |
| 102: More hacking on 'foo'. |
| 103: Finish hacking on 'foo'. |
| |
| The command: |
| |
| fx gerrit-submit 103 |
| |
| will: |
| |
| 1. Add a CQ+1 vote to all the CLs, to start testing them. |
| 2. Add a CQ+2 vote for the first CL, and wait for it to be submitted. |
| 3. Add a CQ+2 vote for the second CL, and wait for it to be submitted. |
| 4. Add a CQ+2 vote for the last CL, and wait for it to be submitted. |
| |
| Adding a CQ+1 to every CL at the beginning speeds up submission: CQ won't |
| need to re-test intermediate CLs if they are not modified in the meantime. |
| |
| If any CL is not ready to submit (for example, it is missing a vote, or has |
| unresolved comments), the tool will abort early. |
| |
| By default, the tool will use the "fuchsia-review.googlesource.com" Gerrit |
| instances. Other instances can be specified using the "--host" parameter: |
| |
| fx gerrit-submit --host myteam-review.googlesource.com 12345 |
| |
| """ |
| parser = argparse.ArgumentParser( |
| description=description, |
| formatter_class=argparse.RawDescriptionHelpFormatter, |
| ) |
| parser.add_argument( |
| "cl", |
| metavar="CL", |
| help="Gerrit CL to submit. May either be a CL " |
| "number or Gerrit Change-ID.", |
| ) |
| parser.add_argument( |
| "--host", |
| dest="host", |
| help="Gerrit host to connect to. " |
| 'Defaults to "fuchsia-review.googlesource.com".', |
| default="fuchsia-review.googlesource.com", |
| ) |
| parser.add_argument( |
| "--num-retries", |
| metavar="N", |
| type=int, |
| default=0, |
| help="number of times to retry a failed submission. " "Defaults to 0.", |
| ) |
| parser.add_argument( |
| "-n", |
| "--dry-run", |
| action="store_true", |
| help="If specified, show the set of CLs that would " |
| "be submitted, but don" |
| "t actually submit.", |
| ) |
| parser.add_argument( |
| "-t", |
| "--batch", |
| action="store_true", |
| help="If specified, don" "t prompt before starting " "submit.", |
| ) |
| parser.add_argument( |
| "--ignore-comments", |
| action="store_true", |
| help="If specified, allow submission of CLs that still " |
| "have unresolved comments on them.", |
| ) |
| return parser.parse_args() |
| |
| |
| def is_valid_change_id(value: str) -> bool: |
| """Determine if "value" is a valid Gerrit CL identifier.""" |
| |
| # Match strings of the form I609f446e6721dd95624939dd041189052054fb83 |
| if re.fullmatch("I[a-f0-9]{8,}", value): |
| return True |
| |
| # Match plain integer change IDs. |
| if re.fullmatch("[1-9][0-9]*", value): |
| return True |
| |
| return False |
| |
| |
| def should_continue() -> bool: |
| """Prompt the user to determine if an action should continue.""" |
| val = input("Submit CLs? [y/N] ") |
| if val.lower()[:1] == "y": |
| return True |
| |
| print("Aborting.") |
| return False |
| |
| |
| def main() -> int: |
| # Parse and validate arguments. |
| args = parse_args() |
| |
| # Ensure the Change-ID looks like a Gerrit Change-ID, and not a git commit. |
| # |
| # TODO: Support Git commit ids and ranges as well. |
| if not is_valid_change_id(args.cl): |
| print( |
| "The argument '%s' does not look like a valid Gerrit Change ID." |
| % args.cl |
| ) |
| print() |
| print( |
| "Please provide either the numeric CL number (such as 12345) or\n" |
| "the Change-ID found at the bottom of the CL description (such as\n" |
| "'I0123456789abcdef0123456789abcdef0123456789abcdef'" |
| ) |
| return 1 |
| |
| # Check we have valid authentication tokens. |
| gerrit_util.EnsureAuthenticated(args.host) |
| |
| # Get the specified change, and bail out if we can't find it. |
| server = GerritServer(args.host) |
| specified_change = server.fetch_change(args.cl) |
| if specified_change is None: |
| print("Could not find Gerrit commit with id '%s'." % args.cl) |
| return 1 |
| |
| # Fetch the list of dependent CLs, and reverse into the order we need to |
| # submit it in. |
| change_ids = server.get_change_dependencies(specified_change.change_id) |
| change_ids.reverse() |
| |
| # Fetch CL details. |
| changes = [] |
| for change_id in change_ids: |
| change = server.fetch_change(change_id) |
| if change is None: |
| print("Could not find CL with id '%s'." % change_id) |
| return 1 |
| changes.append(change) |
| |
| # Print the set of changes. |
| print_changes(changes) |
| |
| # Ensure we can submit the chain. |
| ensure_changes_submittable( |
| changes, abort_on_unresolved_comments=not args.ignore_comments |
| ) |
| |
| # Stop here on dry-run. |
| if args.dry_run: |
| return 0 |
| |
| # Ask user to confirm. |
| if not args.batch and not should_continue(): |
| return 0 |
| |
| # Submit the changes. |
| success = submit_changes( |
| util.Clock(), server, changes, num_retries=args.num_retries |
| ) |
| if not success: |
| return 1 |
| |
| # Success |
| print() |
| print("All changes submitted.") |
| return 0 |
| |
| |
| if __name__ == "__main__": |
| try: |
| sys.exit(main()) |
| except SubmitError as e: |
| print("Error: %s" % e.message) |
| except gerrit_util.GerritError as e: |
| print("Error: %s" % e.message) |
| sys.exit(1) |
