| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """ |
| Recipe to build GCE images with Packer. |
| """ |
| |
| from recipe_engine.recipe_api import Property |
| |
| import re |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| DEPS = [ |
| "fuchsia/build_input_resolver", |
| "fuchsia/git", |
| "fuchsia/status_check", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/futures", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = { |
| "repo": Property(kind=str, help="Salt repository to checkout."), |
| "dry_run": Property( |
| kind=bool, help="Exit early instead of creating a disk image.", default=True |
| ), |
| } |
| |
| |
| def RunSteps(api, repo, dry_run): |
| # Resolve the build input to always contain a Gitiles commit. |
| api.build_input_resolver.resolve(default_project_url=repo) |
| |
| salt_path = api.path["start_dir"].join("salt") |
| revision = api.git.checkout_from_build_input( |
| path=salt_path, repo=repo, rebase_merges=True |
| ) |
| |
| # Get a short revision, image names must be < 64 characters |
| with api.context(cwd=salt_path): |
| revision = api.git( |
| "git rev-parse", |
| "rev-parse", |
| "--short", |
| revision, |
| stdout=api.raw_io.output_text(), |
| ).stdout.rstrip() |
| |
| template_path = salt_path.join("packer", "template.json") |
| |
| env = { |
| # Disable update checks. |
| "CHECKPOINT_DISABLE": "1", |
| # Enable verbose logging. |
| "PACKER_LOG": "1", |
| # Disable color in logging. |
| "PACKER_NO_COLOR": "1", |
| } |
| |
| with api.context(env=env): |
| api.step( |
| "packer validate", |
| [ |
| "packer", |
| "validate", |
| "-var", |
| "revision={}".format(revision), |
| template_path, |
| ], |
| ) |
| |
| template = api.file.read_json( |
| name="load packer template", |
| source=template_path, |
| ) |
| |
| builds = [] |
| for build in template["builders"]: |
| builds.append( |
| api.futures.spawn( |
| _do_packer_builder, |
| api, |
| build["name"], |
| revision, |
| dry_run, |
| template_path, |
| ) |
| ) |
| |
| api.futures.wait(builds) |
| |
| if any([not build.result() for build in builds]): |
| raise api.step.StepFailure("BUILDS FAILED") |
| |
| |
| def _do_packer_builder(api, build, revision, dry_run, template_path): |
| with api.step.nest(build): |
| result = api.step( |
| "packer build", |
| [ |
| "packer", |
| "build", |
| "-var", |
| "revision={}".format(revision), |
| "-var", |
| "dry_run={}".format(str(dry_run).lower()), |
| "-var", |
| "use_internal_ip=true", |
| "-only={}".format(build), |
| template_path, |
| ], |
| stdout=api.raw_io.output_text(), |
| ok_ret="any", |
| # By default recipe_engine assigns a `cost` of 500 mCPU per step, |
| # this limits our parallelism to 2*NUM_CORES but these steps are |
| # simply waiting 99% of the time we can run far more in parallel. |
| cost=None, |
| ) |
| output = result.stdout |
| result.presentation.logs["output"] = output.splitlines() |
| |
| if result.retcode != 0: |
| if dry_run: |
| for line in output.splitlines(): |
| if "DRYRUN SUCCEEDED" in line: |
| result.presentation.step_text = "DRYRUN SUCCEEDED" |
| return True |
| result.presentation.step_text = "DRYRUN FAILED" |
| result.presentation.status = api.step.FAILURE |
| result.presentation.status = api.step.FAILURE |
| else: |
| return True |
| if result.presentation.status == api.step.FAILURE: |
| failures_regex = re.compile( |
| "(\s*ID:\s(.*?\n).*?Result:\sFalse\n.*?Changes:.*?)\n\s*{}:\s-{{10}}".format( |
| build |
| ), |
| re.DOTALL | re.MULTILINE, |
| ) |
| for f in re.findall(failures_regex, output): |
| result.presentation.logs[f[1]] = f[0].splitlines() |
| return False |
| |
| |
| def GenTests(api): |
| state_failures = """ |
| fail: ---------- |
| fail: ID: /etc/systemd/system/gce-provider-start-agent.service |
| fail: Function: file.managed |
| fail: Result: False |
| fail: Comment: The following requisites were not found: |
| fail: require: |
| fail: user: swarming_use |
| fail: Started: 01:13:41.861499 |
| fail: Duration: 0.028 ms |
| fail: Changes: |
| fail: ---------- |
| fail: ID: gce-provider-start-agent |
| fail: Function: service.enabled |
| fail: Result: False |
| fail: Comment: One or more requisite failed: luci.gce_provider./etc/systemd/system/gce-provider-start-agent.service |
| fail: Started: 01:13:41.862762 |
| fail: Duration: 0.017 ms |
| fail: Changes: |
| fail: ---------- |
| fail: ID: curl |
| fail: Function: pkg.installed |
| fail: Result: True |
| fail: Comment: All specified packages are already installed |
| fail: Started: 01:13:41.868646 |
| fail: Duration: 688.753 ms |
| fail: Changes: |
| fail: ---------- |
| """ |
| |
| repo = "https://fuchsia.googlesource.com/infra/salt" |
| yield ( |
| api.status_check.test("ci_failure", status="failure") |
| + api.buildbucket.ci_build(git_repo=repo) |
| + api.properties(repo=repo, dry_run=False) |
| + api.step_data( |
| "load packer template", |
| api.file.read_json( |
| json_content={"builders": [{"name": "pass"}, {"name": "fail"}]} |
| ), |
| ) |
| + api.step_data("pass.packer build", retcode=0) |
| + api.step_data( |
| "fail.packer build", |
| api.raw_io.stream_output_text(state_failures), |
| retcode=1, |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("ci_success") |
| + api.buildbucket.ci_build(git_repo=repo) |
| + api.properties(repo=repo, dry_run=False) |
| + api.step_data( |
| "load packer template", |
| api.file.read_json(json_content={"builders": [{"name": "pass"}]}), |
| ) |
| + api.step_data("pass.packer build", retcode=0) |
| ) |
| |
| yield ( |
| api.status_check.test("try_failure", status="failure") |
| + api.buildbucket.try_build(project="infra/salt", git_repo=repo) |
| + api.build_input_resolver.set_gerrit_branch() |
| + api.properties(repo=repo, dry_run=True) |
| + api.step_data( |
| "load packer template", |
| api.file.read_json( |
| json_content={"builders": [{"name": "pass"}, {"name": "fail"}]} |
| ), |
| ) |
| + api.step_data("pass.packer build", retcode=0) |
| + api.step_data( |
| "fail.packer build", |
| api.raw_io.stream_output_text(state_failures), |
| retcode=1, |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("try_success") |
| + api.buildbucket.try_build(project="infra/salt", git_repo=repo) |
| + api.build_input_resolver.set_gerrit_branch() |
| + api.properties(repo=repo, dry_run=True) |
| + api.step_data( |
| "load packer template", |
| api.file.read_json(json_content={"builders": [{"name": "pass"}]}), |
| ) |
| + api.step_data( |
| "pass.packer build", |
| api.raw_io.stream_output_text("DRYRUN SUCCEEDED"), |
| retcode=1, |
| ) |
| ) |