| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for testing LUCI configs.""" |
| |
| import re |
| |
| from recipe_engine import post_process |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import builder as builder_pb2 |
| from PB.go.chromium.org.luci.buildbucket.proto import project_config as bb_pb2 |
| from PB.go.chromium.org.luci.cq.api.config.v2 import cq as cq_pb2 |
| from PB.recipes.fuchsia.luci_config import InputProperties |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/git", |
| "fuchsia/luci_config", |
| "fuchsia/sso", |
| "recipe_engine/cipd", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| NEW_BUILDERS_IN_CQ_MSG = """\ |
| The following builder(s) are referenced in commit-queue.cfg but aren't yet |
| known to Buildbucket. Create the builder(s) in a separate change before adding |
| them to commit-queue.cfg. |
| """ |
| |
| CQ_BUILDERS_DELETED_MSG = """\ |
| The following builder(s) are running in CQ but missing from |
| cr-buildbucket.cfg. Remove the builder(s) from CQ in a separate change before |
| deleting them. |
| """ |
| |
| # lucicfg version used when the `lucicfg_ensure_file` property is not set. |
| # TODO(olivernewman): Convert each repository that uses this recipe to declare a |
| # compatible lucicfg version in a cipd.ensure checked into the repo that this |
| # recipe can use. Then remove support for installing a default version. |
| DEFAULT_LUCICFG_VERSION = "git_revision:735bda925c07e16382897e9786a3761b66eb6998" |
| LUCICFG_CIPD_PACKAGE = "infra/tools/luci/lucicfg/${platform}" |
| |
| |
| def RunSteps(api, props): |
| assert props.starlark_paths, "starlark_paths must not be empty" |
| |
| checkout_root = api.path["start_dir"] |
| api.git.checkout_from_build_input(checkout_root, api.sso.sso_to_https(props.remote)) |
| |
| if props.lucicfg_ensure_file: |
| # TODO(olivernewman): The cleanup directory is supposed to already |
| # exist, so we shouldn't need to re-ensure it in order to call |
| # `mkdtemp()`. Figure out why the cleanup directory doesn't already |
| # exist, and remove this step if possible. |
| api.file.ensure_directory("ensure cleanup dir", api.path["cleanup"]) |
| lucicfg_dir = api.path.mkdtemp("lucicfg") |
| api.cipd.ensure( |
| lucicfg_dir, |
| checkout_root.join(props.lucicfg_ensure_file), |
| "install lucicfg", |
| ) |
| # Assumes that the ensure file installs lucicfg in the root of the CIPD |
| # directory. |
| lucicfg_path = lucicfg_dir.join("lucicfg") |
| else: |
| lucicfg_path = api.cipd.ensure_tool( |
| LUCICFG_CIPD_PACKAGE, DEFAULT_LUCICFG_VERSION |
| ) |
| |
| with api.step.nest("validate"), api.step.defer_results(): |
| for relative_path in props.starlark_paths: |
| abs_path = checkout_root.join(relative_path) |
| api.step( |
| str(relative_path), |
| [lucicfg_path, "validate", "-strict", "-fail-on-warnings", abs_path], |
| ) |
| |
| # The LUCI Config service tracks the "main" branch of all config repos, so |
| # we should skip the race condition check if we're running on a branch other |
| # than "main" because: |
| # 1. Comparing the local (non-main) configs to the deployed (main) configs |
| # will likely return spurious failures, e.g. if the local |
| # commit-queue.cfg file references builders that have been deleted at |
| # HEAD. |
| # 2. It doesn't matter if there are any race conditions in config changes |
| # on non-main branches, because the changes won't be deployed to LUCI |
| # Config anyway; config changes on non-main branches are only necessary |
| # to modify generated files that are consumed by other parts of the |
| # infrastructure. |
| if props.generated_dir and api.buildbucket_util.triggering_branch == "main": |
| with api.step.nest("check for config deployment race conditions"): |
| check_for_deployment_race_conditions( |
| api, checkout_root.join(props.generated_dir) |
| ) |
| |
| |
| def check_for_deployment_race_conditions(api, generated_dir): |
| """ |
| There are some types of LUCI config changes that introduce race |
| conditions in the process of deploying the updated config files to their |
| corresponding services. |
| |
| For example, it's not safe to simultaneously create a new builder and add |
| it to CQ, because CQ might ingest the updated configs before Buildbucket |
| and try to trigger the builder before Buildbucket even knows it exists. |
| This causes CQ to error out and add confusing comments to CLs. |
| |
| To avoid this specific race condition, we compare the modified version of |
| commit-queue.cfg with the live version of cr-buildbucket.cfg to make sure |
| that even if CQ ingests the updated configs before Buildbucket does, CQ |
| won't try to trigger any builders that don't yet exist. |
| |
| We also check that the live version of commit-queue.cfg doesn't reference |
| any builders that will soon be deleted, which would cause the same issue |
| if Buildbucket ingests the config changes before CQ does. |
| |
| There are other race conditions that we don't bother checking here; for |
| example, it's technically not safe to immediately start running a new |
| builder in CI by adding it to luci-scheduler.cfg. However, LUCI scheduler |
| trigger failures don't produce user-visible error messages like CQ |
| trigger failures, so it's not as big a deal. Also, adding new builders to |
| CI is a very common workflow (adding builders to CQ is somewhat more |
| rare) and it would be annoying if that always required two separate |
| changes. |
| """ |
| api.path.mock_add_directory(generated_dir.join("fuchsia")) |
| api.path.mock_add_directory(generated_dir.join("fuchsia_internal")) |
| |
| # Our luci configs are set up such that each LUCI project's config files |
| # are in a subdirectory of the "generated" directory whose name is the same |
| # as the project name. |
| project_dirs = [ |
| path |
| for path in api.file.listdir( |
| "find projects", |
| generated_dir, |
| test_data=["fuchsia", "fuchsia_internal", "foo.txt"], |
| ) |
| if api.path.isdir(path) |
| ] |
| |
| all_live_builders = set() |
| all_local_builders = set() |
| |
| for project_dir in project_dirs: |
| project = api.path.basename(project_dir) |
| |
| live_bb_config = api.luci_config.buildbucket(project=project) |
| all_live_builders.update(all_builder_names(api, project, live_bb_config)) |
| |
| local_bb_config = api.luci_config.buildbucket( |
| project=project, local_dir=project_dir.join("luci") |
| ) |
| all_local_builders.update(all_builder_names(api, project, local_bb_config)) |
| |
| for project_dir in project_dirs: |
| project = api.path.basename(project_dir) |
| |
| cq_config_path = project_dir.join("luci", "commit-queue.cfg") |
| api.path.mock_add_paths(cq_config_path) |
| if not api.path.exists(cq_config_path): # pragma: no cover |
| # Skip projects that don't have CQ set up. |
| continue |
| |
| local_cq_config = api.luci_config.commit_queue( |
| project=project, local_dir=project_dir.join("luci") |
| ) |
| |
| # Make sure that the new CQ config won't try to trigger non-existent builders. |
| nonexistent_builders = nonexistent_cq_builders( |
| local_cq_config, all_live_builders |
| ) |
| if nonexistent_builders: |
| msg = NEW_BUILDERS_IN_CQ_MSG + "".join( |
| ["\n- %s" % b for b in nonexistent_builders] |
| ) |
| raise api.step.StepFailure(msg) |
| |
| live_cq_config = api.luci_config.commit_queue(project=project) |
| # Make sure that the existing CQ config won't try to trigger builders |
| # that we're deleting. |
| nonexistent_builders = nonexistent_cq_builders( |
| live_cq_config, all_local_builders |
| ) |
| if nonexistent_builders: |
| msg = CQ_BUILDERS_DELETED_MSG + "".join( |
| ["\n- %s" % b for b in nonexistent_builders] |
| ) |
| raise api.step.StepFailure(msg) |
| |
| |
| def all_builder_names(api, project_name, bb_config): |
| for bucket in bb_config.buckets: |
| for builder in bucket.swarming.builders: |
| yield api.buildbucket_util.full_builder_name( |
| builder_pb2.BuilderID( |
| project=project_name, |
| bucket=bucket.name, |
| builder=builder.name, |
| ) |
| ) |
| |
| |
| def nonexistent_cq_builders(cq_config, all_known_builders): |
| """Checks that all builders in cq_config are known to Buildbucket. |
| |
| Returns the names of any CQ builders that aren't in the list of builders |
| known to Buildbucket. |
| """ |
| nonexistent_builders = set() |
| for group in cq_config.config_groups: |
| for builder in group.verifiers.tryjob.builders: |
| # Includable-only builders are only run if a special footer is in a |
| # change's commit message. Usage of the footer is rare enough that |
| # we're willing to allow errors in that case, in exchange for making |
| # it easier to add and remove builders from CQ. |
| if builder.includable_only: |
| continue |
| if builder.name not in all_known_builders: |
| nonexistent_builders.add(builder.name) |
| return sorted(nonexistent_builders) |
| |
| |
| def GenTests(api): |
| remote = "https://fuchsia.googlesource.com/infra/config" |
| |
| def properties(**kwargs): |
| return api.properties( |
| InputProperties( |
| remote=remote, |
| starlark_paths=["main.star", "dev.star"], |
| generated_dir="generated", |
| **kwargs |
| ) |
| ) |
| |
| nesting = "check for config deployment race conditions" |
| |
| def cq_config(builders, includable_only=False): |
| return cq_pb2.Config( |
| config_groups=[ |
| dict( |
| verifiers=dict( |
| tryjob=dict( |
| builders=[ |
| dict(name=builder_name, includable_only=includable_only) |
| for builder_name in builders |
| ] |
| ) |
| ) |
| ) |
| ] |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("starlark", git_repo=remote) |
| # Cover the logic for installing lucicfg from a checked-in CIPD ensure |
| # file. |
| + properties(lucicfg_ensure_file="cipd.ensure") |
| # Cover the logic that reads builder names from cr-buildbucket.cfg. |
| + api.luci_config.mock_local_config( |
| "fuchsia", |
| "cr-buildbucket.cfg", |
| bb_pb2.BuildbucketCfg( |
| buckets=[ |
| dict( |
| name="try", |
| swarming=dict(builders=[dict(name="core.x64-asan")]), |
| ) |
| ] |
| ), |
| nesting=nesting, |
| ) |
| ) |
| |
| # The second file should be validated even if the first one fails. |
| yield ( |
| api.buildbucket_util.test( |
| "first_file_invalid", tryjob=True, git_repo=remote, status="failure" |
| ) |
| + properties() |
| + api.step_data("validate.main.star", retcode=1) |
| + api.post_process(post_process.MustRun, "validate.dev.star") |
| # Make sure we patch in the triggering change, since this is a tryjob. |
| + api.post_process(post_process.MustRun, "checkout.git rebase") |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "cq_new_builder", tryjob=True, git_repo=remote, status="failure" |
| ) |
| + properties() |
| + api.luci_config.mock_local_config( |
| "fuchsia", |
| "commit-queue.cfg", |
| cq_config(["fuchsia/try/core.x64-asan"]), |
| nesting=nesting, |
| ) |
| + api.post_process( |
| post_process.ResultReasonRE, re.escape(NEW_BUILDERS_IN_CQ_MSG) |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test( |
| "deleting_cq_builder", tryjob=True, git_repo=remote, status="failure" |
| ) |
| + properties() |
| + api.luci_config.mock_config( |
| "fuchsia", |
| "commit-queue.cfg", |
| cq_config(["fuchsia/try/core.x64-asan"]), |
| nesting=nesting, |
| ) |
| + api.post_process( |
| post_process.ResultReasonRE, re.escape(CQ_BUILDERS_DELETED_MSG) |
| ) |
| ) |
| |
| # It should be considered safe to delete an includable_only builder even if |
| # it's still referenced in commit-queue.cfg. |
| yield ( |
| api.buildbucket_util.test( |
| "deleting_includable_only_builder", tryjob=True, git_repo=remote |
| ) |
| + properties() |
| + api.luci_config.mock_config( |
| "fuchsia", |
| "commit-queue.cfg", |
| cq_config(["fuchsia/try/core.x64-asan"], includable_only=True), |
| nesting=nesting, |
| ) |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("non_main_branch", tryjob=True) |
| + properties() |
| + api.buildbucket_util.tryjob_branch("non-main-branch") |
| # Race condition check should be skipped on branches other than "main". |
| + api.post_process( |
| post_process.DoesNotRun, "check for config deployment race conditions" |
| ) |
| ) |