recipes/luci_config.py

# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for testing LUCI configs."""

import re

from recipe_engine import post_process
from recipe_engine.recipe_api import Property

from PB.go.chromium.org.luci.buildbucket.proto import builder as builder_pb2
from PB.go.chromium.org.luci.buildbucket.proto import project_config as bb_pb2
from PB.go.chromium.org.luci.cq.api.config.v2 import cq as cq_pb2

DEPS = [
    "fuchsia/buildbucket_util",
    "fuchsia/checkout",
    "fuchsia/luci_config",
    "fuchsia/status_check",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = {
    "config_project": Property(
        kind=str, help="Jiri remote manifest project containing the luci configs"
    ),
    "manifest": Property(kind=str, help="Jiri manifest to use"),
    "remote": Property(kind=str, help="Remote manifest repository"),
    "starlark_paths": Property(kind=list, help="Starlark file paths to validate"),
    "lucicfg_ensure_file": Property(
        kind=str,
        help=(
            "CIPD ensure file containing the pinned lucicfg tool version, relative to "
            "the integration repo"
        ),
    ),
    "generated_dir": Property(
        kind=str,
        help=(
            "Relative directory within the checkout containing generated files for all "
            "projects"
        ),
    ),
}

NEW_BUILDERS_IN_CQ_MSG = """\
The following builder(s) are referenced in commit-queue.cfg but aren't yet
known to Buildbucket. Create the builder(s) in a separate change before adding
them to commit-queue.cfg.
"""

CQ_BUILDERS_DELETED_MSG = """\
The following builder(s) are running in CQ but missing from
cr-buildbucket.cfg. Remove the builder(s) from CQ in a separate change before
deleting them.
"""


def RunSteps(
    api,
    config_project,
    manifest,
    remote,
    starlark_paths,
    lucicfg_ensure_file,
    generated_dir,
):
    assert starlark_paths, "starlark_paths must not be empty"

    checkout_root = api.path["start_dir"]
    with api.context(infra_steps=True):
        api.checkout.with_options(
            path=checkout_root,
            manifest=manifest,
            remote=remote,
            project=config_project,
        )

    integration_dir = checkout_root.join(
        api.checkout.project(config_project, checkout_root)["path"]
    )

    lucicfg_dir = api.path.mkdtemp("lucicfg")
    api.cipd.ensure(
        lucicfg_dir, integration_dir.join(lucicfg_ensure_file), "install lucicfg"
    )
    lucicfg_path = lucicfg_dir.join("lucicfg")

    with api.step.nest("validate"), api.step.defer_results():
        for relative_path in starlark_paths:
            abs_path = integration_dir.join(relative_path)
            api.step(
                relative_path,
                [lucicfg_path, "validate", "-strict", "-fail-on-warnings", abs_path],
            )

    with api.step.nest("check for config deployment race conditions"):
        check_for_deployment_race_conditions(api, integration_dir.join(generated_dir))


def check_for_deployment_race_conditions(api, generated_dir):
    """
    There are some types of LUCI config changes that introduce race
    conditions in the process of deploying the updated config files to their
    corresponding services.

    For example, it's not safe to simultaneously create a new builder and add
    it to CQ, because CQ might ingest the updated configs before Buildbucket
    and try to trigger the builder before Buildbucket even knows it exists.
    This causes CQ to error out and add confusing comments to CLs.

    To avoid this specific race condition, we compare the modified version of
    commit-queue.cfg with the live version of cr-buildbucket.cfg to make sure
    that even if CQ ingests the updated configs before Buildbucket does, CQ
    won't try to trigger any builders that don't yet exist.

    We also check that the live version of commit-queue.cfg doesn't reference
    any builders that will soon be deleted, which would cause the same issue
    if Buildbucket ingests the config changes before CQ does.

    There are other race conditions that we don't bother checking here; for
    example, it's technically not safe to immediately start running a new
    builder in CI by adding it to luci-scheduler.cfg. However, LUCI scheduler
    trigger failures don't produce user-visible error messages like CQ
    trigger failures, so it's not as big a deal. Also, adding new builders to
    CI is a very common workflow (adding builders to CQ is somewhat more
    rare) and it would be annoying if that always required two separate
    changes.
    """
    # Our luci configs are set up such that each LUCI project's config files
    # are in a subdirectory of the "generated" directory whose name is the same
    # as the project name.
    project_dirs = api.file.listdir(
        "find projects", generated_dir, test_data=["fuchsia", "fuchsia_internal"]
    )

    all_live_builders = set()
    all_local_builders = set()

    for project_dir in project_dirs:
        project = api.path.basename(project_dir)

        live_bb_config = api.luci_config.buildbucket(project=project)
        all_live_builders.update(all_builder_names(api, project, live_bb_config))

        local_bb_config = api.luci_config.buildbucket(
            project=project, local_dir=project_dir.join("luci")
        )
        all_local_builders.update(all_builder_names(api, project, local_bb_config))

    for project_dir in project_dirs:
        project = api.path.basename(project_dir)

        cq_config_path = project_dir.join("luci", "commit-queue.cfg")
        api.path.mock_add_paths(cq_config_path)
        if not api.path.exists(cq_config_path):  # pragma: no cover
            # Skip projects that don't have CQ set up.
            continue

        local_cq_config = api.luci_config.commit_queue(
            project=project, local_dir=project_dir.join("luci")
        )

        # Make sure that the new CQ config won't try to trigger non-existent builders.
        nonexistent_builders = nonexistent_cq_builders(
            local_cq_config, all_live_builders
        )
        if nonexistent_builders:
            msg = NEW_BUILDERS_IN_CQ_MSG + "".join(
                ["\n- %s" % b for b in nonexistent_builders]
            )
            raise api.step.StepFailure(msg)

        live_cq_config = api.luci_config.commit_queue(project=project)
        # Make sure that the existing CQ config won't try to trigger builders
        # that we're deleting.
        nonexistent_builders = nonexistent_cq_builders(
            live_cq_config, all_local_builders
        )
        if nonexistent_builders:
            msg = CQ_BUILDERS_DELETED_MSG + "".join(
                ["\n- %s" % b for b in nonexistent_builders]
            )
            raise api.step.StepFailure(msg)


def all_builder_names(api, project_name, bb_config):
    for bucket in bb_config.buckets:
        for builder in bucket.swarming.builders:
            yield api.buildbucket_util.full_builder_name(
                builder_pb2.BuilderID(
                    project=project_name, bucket=bucket.name, builder=builder.name,
                )
            )


def nonexistent_cq_builders(cq_config, all_known_builders):
    """Checks that all builders in cq_config are known to Buildbucket.

    Returns the names of any CQ builders that aren't in the list of builders
    known to Buildbucket.
    """
    nonexistent_builders = set()
    for group in cq_config.config_groups:
        for builder in group.verifiers.tryjob.builders:
            if builder.name not in all_known_builders:
                nonexistent_builders.add(builder.name)
    return sorted(nonexistent_builders)


def GenTests(api):
    properties = api.properties(
        config_project="integration",
        manifest="manifest/infra",
        remote="https://fuchsia.googlesource.com/manifest",
        starlark_paths=["main.star", "dev.star"],
        lucicfg_ensure_file="cipd.ensure",
        generated_dir="generated",
    )

    nesting = "check for config deployment race conditions"

    def cq_config(builders):
        return cq_pb2.Config(
            config_groups=[
                dict(
                    verifiers=dict(
                        tryjob=dict(
                            builders=[
                                dict(name=builder_name) for builder_name in builders
                            ]
                        )
                    )
                )
            ]
        )

    yield (
        api.status_check.test("starlark")
        + api.buildbucket.try_build(project="fuchsia")
        + properties
        # Cover the logic that reads builder names from cr-buildbucket.cfg.
        + api.luci_config.mock_local_config(
            "fuchsia",
            "cr-buildbucket.cfg",
            bb_pb2.BuildbucketCfg(
                buckets=[
                    dict(
                        name="try",
                        swarming=dict(builders=[dict(name="core.x64-asan")]),
                    )
                ]
            ),
            nesting=nesting,
        )
    )

    # The second file should be validated even if the first one fails.
    yield (
        api.status_check.test("first_file_invalid", status="failure")
        + api.buildbucket.try_build(project="fuchsia")
        + properties
        + api.step_data("validate.main.star", retcode=1)
        + api.post_process(post_process.MustRun, "validate.dev.star")
    )

    yield (
        api.status_check.test("cq_new_builder", status="failure")
        + api.buildbucket.try_build(project="fuchsia")
        + properties
        + api.luci_config.mock_local_config(
            "fuchsia",
            "commit-queue.cfg",
            cq_config(["fuchsia/try/core.x64-asan"]),
            nesting=nesting,
        )
        + api.post_process(
            post_process.ResultReasonRE, re.escape(NEW_BUILDERS_IN_CQ_MSG)
        )
    )

    yield (
        api.status_check.test("deleting_cq_builder", status="failure")
        + api.buildbucket.try_build(project="fuchsia")
        + properties
        + api.luci_config.mock_config(
            "fuchsia",
            "commit-queue.cfg",
            cq_config(["fuchsia/try/core.x64-asan"]),
            nesting=nesting,
        )
        + api.post_process(
            post_process.ResultReasonRE, re.escape(CQ_BUILDERS_DELETED_MSG)
        )
    )