# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for closing the tree when builders in a specified Milo console fail."""

import datetime
import hashlib
import re
import textwrap

import attr

from recipe_engine import post_process

from PB.go.chromium.org.luci.buildbucket.proto import (
    build as build_pb2,
    builder_common as builder_common_pb2,
    builds_service as builds_service_pb2,
    common as common_pb2,
)
from PB.go.chromium.org.luci.milo.api.config import project as milo_pb2
from PB.recipe_engine.result import RawResult
from PB.recipes.fuchsia.tree_closer import InputProperties

DEPS = [
    "fuchsia/buildbucket_util",
    "fuchsia/cipd_ensure",
    "fuchsia/issuetracker",
    "fuchsia/luci_config",
    "fuchsia/tree_status",
    "recipe_engine/buildbucket",
    "recipe_engine/context",
    "recipe_engine/json",
    "recipe_engine/properties",
    "recipe_engine/raw_io",
    "recipe_engine/step",
    "recipe_engine/time",
]

PROPERTIES = InputProperties

DEFAULT_GRACE_PERIOD = datetime.timedelta(hours=2)

# The format used for the "date" field in HTTP responses from requests to the
# tree status page.
OLD_TREE_STATUS_DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
TREE_STATUS_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"

MESSAGE_SUFFIX = " (automatic)"


def RunSteps(api, props):
    now = los_angeles_now(api)

    closing_disabled_reason = None

    if not during_working_hours(now):
        closing_disabled_reason = "outside working hours"

    for date in props.disable_on_dates:
        if now.date() == datetime.date.fromisoformat(date):
            closing_disabled_reason = f"disabled today ({date})"
            break

    grace_period = (
        datetime.timedelta(seconds=props.grace_period_seconds) or DEFAULT_GRACE_PERIOD
    )

    auto_closer_username = api.buildbucket.build.builder.builder

    with api.context(infra_steps=True):
        status = api.tree_status.get(props.tree_status_host)
        if props.tree_name:
            old_status = status
            status = api.tree_status.get("", tree_name=props.tree_name)
            # If the tree status was manually set, then the two apps will be out
            # of sync, so copy the newer status to the other app.
            # TODO(https://fxbug.dev/332741591): Remove when we no longer need
            # the old app for the stats dashboards.
            if old_status.message != status.message:
                if old_status.date > status.date:
                    api.tree_status.update(
                        message=old_status.message,
                        hostname=props.tree_status_host,
                        admin_hostname=props.tree_status_admin_host,
                        tree_name=props.tree_name,
                        username=auto_closer_username,
                        state=old_status.state.value,
                        step_name="copy tree status to new app",
                        last_status=status,
                        update_all=False,
                    )
                    # Retrieve the newly copied status.
                    status = api.tree_status.get("", tree_name=props.tree_name)
                else:
                    api.tree_status.update(
                        message=status.message,
                        hostname=props.tree_status_host,
                        admin_hostname=props.tree_status_admin_host,
                        username=auto_closer_username,
                        state=status.state.value,
                        step_name="copy tree status to old app",
                        last_status=old_status,
                        update_all=False,
                    )

        # Under some conditions the auto-closer is guaranteed not to modify the
        # tree status regardless of the console health, in which case we can
        # exit early.

        if status.open and closing_disabled_reason:
            # If the tree is open and we're during a time that auto-closing is
            # disabled, we should never close the tree.
            return RawResult(
                summary_markdown=closing_disabled_reason,
                status=common_pb2.SUCCESS,
            )

        if not (status.message.endswith(MESSAGE_SUFFIX)) or (
            status.username and status.username != auto_closer_username
        ):
            # If the tree is already closed by a human we shouldn't auto-reopen.
            if not status.open:
                return RawResult(
                    summary_markdown="tree is already manually closed",
                    status=common_pb2.SUCCESS,
                )
            open_duration = api.time.utcnow() - status.date
            # If the tree was manually reopened recently it shouldn't be closed
            # again, to avoid stepping on build gardeners' toes.
            if open_duration < grace_period:
                return RawResult(
                    summary_markdown="tree is recently opened",
                    status=common_pb2.SUCCESS,
                )

        consoles = []
        milo_cfg = api.luci_config.milo(project=props.project)
        for console_name in props.console_names:
            matches = [c for c in milo_cfg.consoles if c.id == console_name]
            assert matches, f"no console with name {console_name!r}"
            consoles.append(matches[0])

        with api.step.nest("check console health"):
            summary_regexp_ignore = [re.compile(r) for r in props.summary_regexp_ignore]
            closure_reason = get_tree_closure_reason(
                api, consoles, props.rules, summary_regexp_ignore
            )

        if not closure_reason:
            if not status.open:
                api.tree_status.update(
                    message="Tree is open" + MESSAGE_SUFFIX,
                    hostname=props.tree_status_host,
                    admin_hostname=props.tree_status_admin_host,
                    tree_name=props.tree_name,
                    username=auto_closer_username,
                    state="OPEN",
                    step_name="open tree",
                    last_status=status,
                )
                # TODO(fxbug.dev/97321): Also close the tree closure bug when
                # reopening the tree.

            # Builders are green enough, so no need to close the tree.
            return RawResult(
                summary_markdown="console is healthy", status=common_pb2.SUCCESS
            )

        # Log the bug title so someone looking at the build results page can
        # understand why the builder is closing the tree.
        api.step.empty(
            closure_reason.bug_title(), status=api.step.FAILURE, raise_on_failure=False
        )

        # The console is unhealthy but the tree is already closed, so no action
        # is necessary.
        if not status.open:
            return RawResult(
                summary_markdown=f"{closure_reason.bug_title()} (tree is already closed)",
                status=common_pb2.SUCCESS,
            )

        try:
            culprit_ranking_text = run_autogardener(
                api, [b.id for b in closure_reason.failed_builds()]
            )
        except api.step.StepFailure:  # pragma: no cover
            # TODO(olivernewman): Raise autogardener exceptions at the end of
            # the build after closing the tree.
            culprit_ranking_text = ""

        with api.step.nest("emit tree_closing_builders") as presentation:
            presentation.properties["tree_closing_builders"] = sorted(
                {b.builder.builder for b in closure_reason.failed_builds()}
            )

        bug_link = file_bug(
            api, closure_reason, props.bug_component_id, culprit_ranking_text
        )

        message = f"Tree is closed: {bug_link}"
        api.tree_status.update(
            message=message + MESSAGE_SUFFIX,
            hostname=props.tree_status_host,
            admin_hostname=props.tree_status_admin_host,
            tree_name=props.tree_name,
            username=auto_closer_username,
            state="CLOSED",
            # Trim the URL to just the ID.
            issuetracker_id=bug_link.replace("https://fxbug.dev/", ""),
            step_name="close tree",
            last_status=status,
        )

        return RawResult(
            summary_markdown=closure_reason.bug_title(), status=common_pb2.FAILURE
        )


class TreeClosureReason:  # pragma: no cover
    def bug_title(self):
        raise NotImplementedError()

    def bug_description(self):
        """Returns the description of the bug to file."""
        raise NotImplementedError()

    def failed_builds(self):
        """Failed Buildbucket builds that triggered the closure.

        Returns: list of build.proto.
        """
        raise NotImplementedError()


@attr.s
class ReasonConsecutiveFailures(TreeClosureReason):
    _builder = attr.ib(type=str)
    _failed_builds = attr.ib(type=[build_pb2.Build])

    def bug_title(self):
        return f"{self._builder} failed {len(self._failed_builds)} times in a row"

    def bug_description(self):
        description = f"""        The tree was closed because {self._builder} failed {len(self._failed_builds)} times in a row.

        Full builder history: {builder_link(self._failed_builds[0].builder, limit=200)}

        Failed builds:

        """
        lines = textwrap.dedent(description).splitlines()

        for build in self._failed_builds:
            lines.append(f"- {build_link(build)}")
            if build.summary_markdown:
                lines.extend(formatted_summary_lines(build))

        return "\n".join(lines)

    def failed_builds(self):
        return self._failed_builds


@attr.s
class ReasonConcurrentFailures(TreeClosureReason):
    _failed_builds_by_builder = attr.ib(type={str: build_pb2.Build})

    def bug_title(self):
        return f"{len(self._failed_builds_by_builder)} builders are failing"

    def bug_description(self):
        description = f"""        The tree was closed because {len(self._failed_builds_by_builder)} builders are failing.

        Failing builders:

        """
        lines = textwrap.dedent(description).splitlines()
        for builder, latest_build in sorted(self._failed_builds_by_builder.items()):
            lines.append(f"- {builder}: {build_link(latest_build)}")
            if latest_build.summary_markdown:
                lines.extend(formatted_summary_lines(latest_build))
        return "\n".join(lines)

    def failed_builds(self):
        return self._failed_builds_by_builder.values()


def build_link(build):
    return f"{builder_link(build.builder)}/b{int(build.id)}"


def builder_link(builder, limit=None):
    url = f"https://ci.chromium.org/p/{builder.project}/builders/{builder.bucket}/{builder.builder}"
    if limit:
        url += f"?limit={int(limit)}"
    return url


def formatted_summary_lines(build):
    lines = [""]
    lines.extend("  > " + line for line in build.summary_markdown.split("\n"))
    lines.append("")
    return lines


def get_tree_closure_reason(api, consoles, rules, summary_regexp_ignore):
    """Determines if and why the tree should be closed.

    Args:
        consoles (seq of milo_pb2.Console): Milo consoles to check the builders of.
        rules (seq of Rule proto): The conditions under which the tree should
            be closed.
        summary_regexp_ignore (seq of re.Pattern): Any build whose summary matches
            one of these regexes will not be considered for the purpose of
            closing the tree.

    Returns:
        A TreeClosureReason if the tree should be closed, otherwise None.
    """
    failing_builders = {}

    concurrent_failures_rules = [r for r in rules if r.concurrent_failures > 0]
    consecutive_failures_rules = [r for r in rules if r.consecutive_failures > 0]

    # Search back in history by the maximum number of builds required by any
    # rule. If no rule specifies a `consecutive_failures` value, then we only
    # need to check the most recent build of each builder for each concurrent
    # failure rule.
    if consecutive_failures_rules:
        count_to_check = max(r.consecutive_failures for r in consecutive_failures_rules)
    else:
        count_to_check = 1

    # Fetch more builds than we actually need in case some match one of
    # `summary_regexp_ignore`.
    count_to_fetch = max(5, count_to_check * 2)

    # TODO(olivernewman): Combine the search requests into a single batch RPC
    # request rather than doing them serially. This will likely be
    # significantly more efficient.
    for console in consoles:
        for builder in console.builders:
            builds = last_n_builds(api, builder, count_to_fetch)
            # Rules only apply to builds that failed, so no need to even iterate
            # over the rules if all of its builds passed.
            if all(build.status == common_pb2.SUCCESS for build in builds):
                continue

            # The buildbucket API is supposed to return builds newest-first, but
            # we'll sort again here just to be safe.
            builds.sort(key=lambda b: b.start_time.seconds, reverse=True)

            builds = [
                b
                for b in builds
                if not any(r.search(b.summary_markdown) for r in summary_regexp_ignore)
            ][:count_to_check]
            if not builds:
                # Either builder has no history, or all of its recent builds match
                # one of summary_regexp_ignore.
                continue

            builder_name = builder.name.split("/")[-1]
            latest_build = builds[0]

            # Likewise, if a builder's most recent build passed, then there's no
            # need to close the tree for this builder.
            if latest_build.status == common_pb2.SUCCESS:
                continue

            builder_name = builder.name.split("/")[-1]
            failing_builders[builder_name] = latest_build

            for rule in consecutive_failures_rules:
                if (
                    rule.builders_to_check
                    and builder_name not in rule.builders_to_check
                ):
                    # We're only checking this rule for a subset of builders, and
                    # this builder is not in that subset.
                    continue

                if len(builds) < rule.consecutive_failures:
                    # This builder doesn't yet have enough history to determine whether
                    # it should be closed.
                    continue

                builds_to_check = builds[: rule.consecutive_failures]
                rule_matched = True
                for build in builds_to_check:
                    if build.status == common_pb2.SUCCESS:
                        rule_matched = False
                        break
                    failed_step = find_failed_step_match(build, rule.failed_step_regexp)
                    if not failed_step:
                        rule_matched = False
                        break

                if rule_matched:
                    return ReasonConsecutiveFailures(
                        builder=builder_name,
                        failed_builds=builds_to_check,
                    )

    for rule in concurrent_failures_rules:
        matched_builders = {}
        # We only consider the most recent build of each builder when checking
        # for concurrently failing builders.
        for builder, latest_build in failing_builders.items():
            failed_step = find_failed_step_match(latest_build, rule.failed_step_regexp)
            if failed_step:
                matched_builders[builder] = latest_build

        if len(matched_builders) >= rule.concurrent_failures:
            return ReasonConcurrentFailures(matched_builders)

    return None


def find_failed_step_match(build, step_regex):
    for step in build.steps:
        if step.status != common_pb2.SUCCESS and re.search(step_regex, step.name):
            return step
    return None


def last_n_builds(api, console_builder, n):
    # builder.name is of the form
    # "buildbucket/luci.{project}.{bucket}/{builder}".
    _, full_bucket, builder_name = console_builder.name.split("/")
    _, project, bucket = full_bucket.split(".", 2)
    predicate = builds_service_pb2.BuildPredicate(
        builder=builder_common_pb2.BuilderID(
            builder=builder_name,
            bucket=bucket,
            project=project,
        ),
        status=common_pb2.ENDED_MASK,  # Include only completed builds.
    )
    fields = api.buildbucket.DEFAULT_FIELDS.union({"steps", "summary_markdown"})
    builds = api.buildbucket.search(
        predicate, limit=n, step_name=builder_name, fields=fields
    )
    return builds


def run_autogardener(api, build_ids):
    """Given IDs of failed buildbucket builds, run culprit analysis.

    Args:
        build_ids (seq of int): List of failing build IDs.

    Returns: A markdown-formatting string containing a ranking of potential
    culprit changes.
    """
    exe = api.cipd_ensure(
        api.resource("cipd.ensure"),
        "fuchsia/infra/autogardener/${platform}",
    )
    cmd = [
        exe,
        "culprit",
        "-json-output",
        api.json.output(),
    ]
    cmd.extend(build_ids)
    return api.step(
        "find culprits",
        cmd,
        step_test_data=lambda: api.json.test_api.output(
            {"markdown_output": "1. foo\n2. bar"}
        ),
        timeout=datetime.timedelta(minutes=10),
    ).json.output["markdown_output"]


def file_bug(api, closure_reason, bug_component_id, culprit_ranking):
    """Every tree closure needs a tracking bug linked from the tree status.

    Returns:
        The bug link.
    """
    description = closure_reason.bug_description()
    if culprit_ranking:
        description += "\n\nCulprit analysis:\n\n" + culprit_ranking
    return api.issuetracker.file_bug(
        "create issuetracker bug",
        closure_reason.bug_title(),
        description,
        component_id=bug_component_id,
    )


def los_angeles_now(api):
    # The `time` recipe module and Python's stdlib don't have good timezone
    # support, so shell out to unix `date` to get the timezone-aware time.
    with api.context(env={"TZ": "America/Los_Angeles"}):
        step = api.step(
            "get current time",
            ["date", "--iso-8601=minutes"],
            stdout=api.raw_io.output_text(add_output_log=True),
            step_test_data=lambda: api.raw_io.test_api.stream_output_text(
                "2021-12-01T09:53"
            ),
        )
    return datetime.datetime.fromisoformat(step.stdout.strip())


def during_working_hours(now):
    """Checks if the current time is within West Coast working hours.

    Working hours are 9 AM to 5 PM, Monday-Friday.
    """
    return 0 <= now.weekday() <= 4 and 9 <= now.hour < 17


def GenTests(api):
    def pseudo_random_build_id(builder_string):
        return int(int(hashlib.sha256(builder_string.encode()).hexdigest(), 16) % 1e8)

    builder_name = "auto-closer"

    def test(
        name,
        builder_history=None,
        should_check_tree=True,
        should_check_console=True,
        should_close_tree=False,
        should_open_tree=False,
        rules=None,
        summary_regexp_ignore=(),
        tree_status="OPEN",
        tree_status_user="human@example.com",
        tree_status_age=DEFAULT_GRACE_PERIOD + datetime.timedelta(minutes=1),
        disable_on_dates=(),
        bug_component_id=None,
        tree_name=None,
    ):
        """Create a test case for running this recipe.

        Args:
            name (str): Name of the test case.
            builder_history (dict): Mapping from builder base name to a list
                of build status strings like "SUCCESS", corresponding to the
                builder's most recent builds in reverse chronological order
                (latest builds first).
            should_check_console (bool): Whether the recipe is expected to
                check the current tree status.
            should_check_console (bool): Whether the recipe is expected to
                query luci-config and Buildbucket for builders and build results.
            should_close_tree (bool): Whether the recipe is expected to close
                the tree. Ignored if should_check_console is False.
            should_open_tree (bool): Whether the recipe is expected to open
                the tree. Ignored if should_check_console is False.
            rules (seq of Rule): Passed as a recipe property.
            summary_regexp_ignore (seq of str): Passed as a recipe property.
            tree_status (str): Mock current status for the tree.
            tree_status_user (str): Username associated with the last tree
                status.
            tree_status_age (str): Mocked duration since the last tree status
                update.
            disable_on_dates (seq of str): Passed as a recipe property.
        """
        if not builder_history:
            builder_history = {}

        if rules is None:
            rules = [
                dict(concurrent_failures=2, consecutive_failures=2),
            ]

        project = "fuchsia"
        bucket = "global.ci"
        builder_name = "auto-closer"

        res = api.buildbucket_util.test(
            name,
            status="FAILURE" if should_close_tree else "SUCCESS",
            project=project,
            builder=builder_name,
        ) + api.properties(
            console_names=["global_ci", "another_ci"],
            rules=rules,
            summary_regexp_ignore=list(summary_regexp_ignore),
            tree_status_host="example.com",
            tree_status_admin_host="admin.example.com",
            bug_component_id=bug_component_id,
            disable_on_dates=list(disable_on_dates),
            tree_name=tree_name,
        )

        if not should_check_tree:
            return res

        now = 1592654400
        res += api.time.seed(now)

        status_date = datetime.datetime.utcfromtimestamp(now) - tree_status_age
        old_app_more_recent = tree_status_user != builder_name
        old_status = tree_status
        if tree_name:
            old_status = "CLOSED" if tree_status == "OPEN" else "OPEN"
        res += api.step_data(
            "get current tree status",
            stdout=api.json.output(
                {
                    "general_state": (
                        tree_status.lower()
                        if old_app_more_recent
                        else old_status.lower()
                    ),
                    "username": tree_status_user,
                    "key": 12345,
                    "date": status_date.strftime(OLD_TREE_STATUS_DATE_FORMAT),
                    "message": f"Tree is {tree_status if old_app_more_recent else old_status}{MESSAGE_SUFFIX}",
                }
            ),
        )
        if tree_name:
            if old_app_more_recent:
                status_date -= datetime.timedelta(seconds=5)
            else:
                status_date += datetime.timedelta(seconds=5)
            res += api.step_data(
                "get current tree status (2)",
                stdout=api.json.output(
                    {
                        "generalState": (
                            old_status if old_app_more_recent else tree_status
                        ),
                        "name": f"trees/{tree_name}/status/12345",
                        "createTime": status_date.strftime(TREE_STATUS_DATE_FORMAT),
                        "message": f"Tree is {old_status if old_app_more_recent else tree_status}{MESSAGE_SUFFIX}",
                    }
                ),
            )
            if old_app_more_recent:
                res += api.step_data(
                    "check for tree status collision",
                    stdout=api.json.output(
                        {
                            "generalState": old_status,
                            "name": f"trees/{tree_name}/status/12345",
                            "createTime": status_date.strftime(TREE_STATUS_DATE_FORMAT),
                            "message": f"Tree is {old_status}{MESSAGE_SUFFIX}",
                        }
                    ),
                )
                res += api.step_data(
                    "get current tree status (3)",
                    stdout=api.json.output(
                        {
                            "generalState": tree_status,
                            "name": f"trees/{tree_name}/status/12345",
                            "createTime": status_date.strftime(TREE_STATUS_DATE_FORMAT),
                            "message": f"Tree is {tree_status}{MESSAGE_SUFFIX}",
                        }
                    ),
                )
            else:
                res += api.step_data(
                    "check for tree status collision",
                    stdout=api.json.output(
                        {
                            "general_state": old_status.lower(),
                            "username": tree_status_user,
                            "key": 12345,
                            "date": (
                                status_date - datetime.timedelta(seconds=5)
                            ).strftime(OLD_TREE_STATUS_DATE_FORMAT),
                            "message": f"Tree is {old_status}{MESSAGE_SUFFIX}",
                        }
                    ),
                )
            if should_close_tree:
                res += api.step_data(
                    "check for tree status collision (2)",
                    stdout=api.json.output(
                        {
                            "generalState": tree_status,
                            "name": f"trees/{tree_name}/status/12345",
                            "createTime": status_date.strftime(TREE_STATUS_DATE_FORMAT),
                            "message": f"Tree is {tree_status}{MESSAGE_SUFFIX}",
                        }
                    ),
                )

        if should_check_console:
            res += api.post_process(post_process.MustRun, "check console health")
        else:
            res += api.post_process(post_process.DoesNotRun, "check console health")
            return res

        checker = post_process.MustRun if should_close_tree else post_process.DoesNotRun
        res += api.post_process(checker, "create issuetracker bug")
        res += api.post_process(checker, "close tree")

        res += api.post_process(
            post_process.MustRun if should_open_tree else post_process.DoesNotRun,
            "open tree",
        )

        console_builders = []
        for builder in builder_history:
            console_builders.append(
                milo_pb2.Builder(name=f"buildbucket/luci.{project}.{bucket}/{builder}")
            )

        milo_cfg = milo_pb2.Project(
            consoles=[
                milo_pb2.Console(
                    id="global_ci",
                    name="global integration ci",
                    repo_url="https://fuchsia.googlesource.com/integration",
                    refs=["regexp:refs/heads/main"],
                    # Split builders between the two consoles to test the
                    # multi-console logic.
                    builders=[console_builders[0]],
                ),
                milo_pb2.Console(
                    id="another_ci",
                    name="another ci",
                    repo_url="https://fuchsia.googlesource.com/other-repo",
                    refs=["regexp:refs/heads/main"],
                    builders=console_builders[1:],
                ),
            ]
        )
        res += api.luci_config.mock_config(project, "luci-milo.cfg", milo_cfg)

        # History is a list of builder statuses, most recent first.
        for builder, history in builder_history.items():
            search_results = []
            for i, build in enumerate(history):
                bp = build.builder
                bp.builder, bp.bucket, bp.project = builder, bucket, project
                build.id = pseudo_random_build_id(builder + str(i))
                build.start_time.seconds = int(
                    now - datetime.timedelta(days=i).total_seconds()
                )
                search_results.append(build)
            res += api.buildbucket.simulated_search_results(
                search_results,
                step_name=f"check console health.{builder}",
            )

        return res

    def _build(build_status, steps=None, summary_markdown=""):
        """Returns a Build proto for mocking buildbucket.search requests.

        Args:
            build_status (str): A string corresponding to a common_pb2.Status
                value.
            steps (dict): A mapping from step names (with nestings indicated
                by | pipes) to status strings, also corresponding to
                common_pb2.Status values.
            summary_markdown (str): The summary to set for the build.
        """
        steps = steps or {}
        b = api.buildbucket.ci_build_message(status=build_status)
        b.summary_markdown = summary_markdown
        for step_name, status in steps.items():
            step = b.steps.add()
            step.name = step_name
            step.status = common_pb2.Status.Value(status)
        return b

    def success(**kwargs):
        """Returns a Build proto with a successful status."""
        return _build("SUCCESS", **kwargs)

    def failure(summary_markdown="5 tests failed", **kwargs):
        """Returns a Build proto with a failure status."""
        return _build("FAILURE", summary_markdown=summary_markdown, **kwargs)

    # As long as the most recent build for each builder is green, we shouldn't
    # close the tree.
    yield test(
        name="all_green",
        builder_history={
            "core.arm64-asan": [success(), failure()],
            "core.x64-asan": [success(), failure()],
        },
        # Non-current dates should be ignored.
        disable_on_dates=["2019-05-06"],
    )

    # If any builder's most recent `consecutive_failures` builds have all failed,
    # close the tree.
    yield test(
        name="consecutive_failures",
        builder_history={
            "core.arm64-asan": [failure(steps={"foo": "FAILURE"}), success()],
            "core.x64-asan": [
                failure(steps={"foo": "FAILURE"}),
                failure(steps={"bar": "SUCCESS", "foo": "FAILURE"}),
            ],
        },
        rules=[
            dict(
                consecutive_failures=2,
                failed_step_regexp="foo",
                builders_to_check=["core.x64-asan"],
            )
        ],
        should_close_tree=True,
        bug_component_id=12345,
        tree_name="fuchsia-stem",
    )

    # Even if there are consecutive failures, ignore them if we're only checking
    # the rule for a subset of builders, and the failing builder is not in that
    # subset.
    yield test(
        name="ignored_consecutive_failures",
        builder_history={
            "core.arm64-asan": [failure(steps={"foo": "FAILURE"}), success()],
            "core.x64-asan": [
                failure(steps={"foo": "FAILURE"}),
                failure(steps={"bar": "SUCCESS", "foo": "FAILURE"}),
            ],
        },
        rules=[
            dict(
                consecutive_failures=2,
                failed_step_regexp="foo",
                builders_to_check=["core.arm64-asan"],
            )
        ],
        should_close_tree=False,
    )

    yield test(
        name="summary_regexp_ignore",
        builder_history={
            "core.x64-asan": [
                failure(
                    summary_markdown="step failed: checkout.jiri update",
                    steps={"foo": "FAILURE"},
                ),
            ],
        },
        rules=[dict(consecutive_failures=1)],
        summary_regexp_ignore=["checkout"],
        should_close_tree=False,
    )

    # Even if a builder has failed more than `consecutive_failures` times in a
    # row, don't close the tree unless all the failed builds have a failed step
    # matching `failed_step_regexp`.
    yield test(
        name="consecutive_failures_no_matched_step",
        builder_history={
            "core.x64-asan": [
                failure(steps={"foo": "FAILURE"}),
                failure(steps={"bar": "FAILURE"}),
            ]
        },
        rules=[dict(consecutive_failures=2, failed_step_regexp="foo")],
        should_close_tree=False,
    )

    # If `concurrent_failures` or more builders' most recent builds have all
    # failed, close the tree.
    yield test(
        name="concurrent_failures",
        builder_history={
            "bringup.arm64-asan": [success()],
            "core.arm64-asan": [failure(steps={"foo": "FAILURE"})],
            "core.x64-asan": [failure(steps={"bar": "SUCCESS", "foo|x": "FAILURE"})],
        },
        rules=[dict(concurrent_failures=2, failed_step_regexp="foo")],
        should_close_tree=True,
    )

    # Even if `concurrent_failures` or more builders' most recent builds have
    # all failed, don't close the tree if they don't all have a failed step
    # matching `failed_step_regexp`.
    yield test(
        name="concurrent_failures_no_matched_step",
        builder_history={
            "bringup.arm64-asan": [failure()],
            "core.arm64-asan": [failure(steps={"bar": "FAILURE"})],
            # Only this builder has a failed step matching `failed_step_regexp`.
            "core.x64-asan": [failure(steps={"bar": "SUCCESS", "foo|x": "FAILURE"})],
        },
        rules=[dict(concurrent_failures=2, failed_step_regexp="foo")],
        should_close_tree=False,
    )

    # The tree status shouldn't be changed if it was only recently opened.
    yield test(
        name="tree_recently_opened",
        tree_status_age=DEFAULT_GRACE_PERIOD - datetime.timedelta(minutes=1),
        should_check_console=False,
    )

    # The build should exit early if the tree is already closed by a human.
    yield test(
        name="tree_closed_by_human", tree_status="CLOSED", should_check_console=False
    )

    # We should not take any action if the tree is already auto-closed and the
    # console remains unhealthy.
    yield test(
        name="tree_already_closed",
        tree_status="CLOSED",
        builder_history={
            "core.x64-asan": [
                failure(steps={"foo": "FAILURE"}),
                failure(steps={"bar": "FAILURE"}),
            ]
        },
        rules=[dict(consecutive_failures=2)],
        tree_status_user=builder_name,
        should_close_tree=False,
        tree_name="fuchsia",
    )

    # The tree should be reopened if it is currently auto-closed and the console
    # is healthy.
    yield test(
        name="reopen",
        tree_status="CLOSED",
        tree_status_user=builder_name,
        builder_history={
            "bringup.arm64-asan": [success()],
            "core.arm64-asan": [success()],
        },
        should_open_tree=True,
    )

    # Builders with no builds shouldn't affect the tree status.
    yield test(
        name="throttled__no_history",
        # If the tree is "throttled" then it's open, so the closer should still
        # check the console for failures.
        tree_status="THROTTLED",
        builder_history={"core.arm64-asan": []},
        should_close_tree=False,
    )

    # Builders with fewer than consecutive_failures builds shouldn't affect the
    # tree status.
    yield test(
        name="not_enough_history",
        builder_history={"core.arm64-asan": [failure()]},
        should_close_tree=False,
    )

    weekday_nighttime = "2021-12-03T23:53"  # Friday at 11:53 PM
    weekend_daytime = "2021-12-04T11:01"  # Saturday at 11:01 AM

    # Tree closer should exit early if the time is outside West Coast working
    # hours and there are no off-hour rules configured.
    yield test("weekend", should_check_tree=False) + api.step_data(
        "get current time", api.raw_io.stream_output_text(weekend_daytime)
    )
    yield test("nighttime", should_check_tree=False) + api.step_data(
        "get current time", api.raw_io.stream_output_text(weekday_nighttime)
    )

    # Tree closer should reopen the tree after an automatic closure, even if
    # it's outside working hours.
    yield (
        test(
            "nighttime_reopen_tree",
            should_check_tree=True,
            tree_status="CLOSED",
            tree_status_user=builder_name,
            builder_history={
                "bringup.arm64-asan": [success()],
                "core.arm64-asan": [success()],
            },
            should_open_tree=True,
        )
        + api.step_data(
            "get current time", api.raw_io.stream_output_text(weekday_nighttime)
        )
    )

    yield test(
        "disable_dates", should_check_tree=False, disable_on_dates=["2021-12-01"]
    ) + api.step_data(
        "get current time", api.raw_io.stream_output_text("2021-12-01T12:00")
    )