| # Copyright 2022 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Generic continuous integration recipe for projects in the Fuchsia ecosystem. |
| |
| This recipe supports projects that need the following steps in their CI jobs: |
| |
| 1. Checkout code. |
| 2. Download additional pinned dependencies not checked into version control. |
| Note that this additional download step will be subject to removal once it |
| can be handled by Git itself, per |
| http://go/git-superproject-source-control-for-fuchsia. |
| 3. Build. |
| 4. Upload build artifacts (images, blobs, drivers, etc.). |
| 5. TODO(olivernewman): Run tests in parallel shards on separate machines. |
| 6. TODO(fxbug.dev/93038): Trigger tests in downstream projects, passing through |
| the uploaded build artifacts. |
| """ |
| |
| from google.protobuf import json_format as jsonpb |
| |
| from PB.recipes.fuchsia.build_test_upload import InputProperties |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| DEPS = [ |
| "fuchsia/buildbucket_util", |
| "fuchsia/checkout", |
| "fuchsia/dpi", |
| "fuchsia/fxt", |
| "fuchsia/git", |
| "recipe_engine/context", |
| "recipe_engine/path", |
| "recipe_engine/properties", |
| "recipe_engine/step", |
| ] |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps(api, props): |
| # Projects should not assume they'll always be checked out at the same |
| # absolute path, so do the checkout in a random temporary directory. |
| checkout_dir = api.path.mkdtemp("checkout") |
| |
| if props.jiri_manifest: |
| api.checkout.with_options( |
| path=checkout_dir, |
| manifest=props.jiri_manifest, |
| remote=props.remote, |
| project=props.jiri_project, |
| ) |
| else: |
| api.git.checkout_from_build_input( |
| repo=props.remote, |
| path=checkout_dir, |
| ) |
| |
| # Compute a semantic version that a project can use as a placeholder value |
| # for operations that require a version number, if the project does not have |
| # a formal release process. |
| # |
| # NOTE: This assumes that the repo at the checkout root is the source of |
| # truth for the entire checkout, which may not be the case for some |
| # Jiri-based checkouts where the source of truth is a nested repository. |
| with api.context(cwd=checkout_dir): |
| revision_count = api.git.rev_list_count("HEAD", test_data="1") |
| placeholder_version = "0.0.0.%s" % revision_count |
| |
| upload_namespace = api.buildbucket_util.id |
| |
| # Any of these variables can be used as a format string in a command's |
| # configured arguments and it will be substituted for the actual value. For |
| # example, "--build-dir={build_dir}" in a command's args will be |
| # replaced with "--build-dir=/actual/path/to/build/dir". |
| # |
| # Where possible, this approach is preferred over requiring commands to |
| # support specific flags as it lets each project's commands be fairly |
| # infrastructure-agnostic and reduces coupling between the project and the |
| # infrastructure. |
| build_dir = api.path.mkdtemp("build") |
| variables = { |
| "build_dir": build_dir, |
| "upload_namespace": upload_namespace, |
| "placeholder_version": placeholder_version, |
| } |
| |
| def run_command(step_name, command_pb): |
| # Not all projects need all possible commands, so skip any un-configured |
| # commands. |
| if not command_pb.path: |
| return None |
| cmd = [checkout_dir.join(*command_pb.path.split("/"))] |
| for arg in command_pb.args: |
| # Perform variable substitutions. |
| cmd.append(arg.format(**variables)) |
| return api.step(step_name, cmd) |
| |
| with api.context(cwd=checkout_dir): |
| run_command("download extra dependencies", props.download_command) |
| |
| run_command("build", props.build_command) |
| |
| # TODO(fxbug.dev/92697): Each upload command should emit a manifest of |
| # files for the recipe to upload, rather than doing the upload itself. |
| run_command("upload", props.upload_command) |
| |
| if props.mos_upload_options.repo_hostname: |
| api.dpi.upload( |
| "upload to MOS-TUF repos", |
| build_dir=build_dir, |
| options=props.mos_upload_options, |
| ) |
| |
| if props.gcs_bucket and props.HasField("fxt_options"): |
| api.fxt.orchestrate_fxt_tests( |
| bucket=props.gcs_bucket, |
| namespace=upload_namespace, |
| options=props.fxt_options, |
| ) |
| |
| |
| def GenTests(api): |
| default_remote = "https://fuchsia.googlesource.com/foo" |
| |
| def properties(**kwargs): |
| props = { |
| "remote": default_remote, |
| "build_command": { |
| "path": "scripts/build.sh", |
| "args": [ |
| "--build-dir", |
| "{build_dir}", |
| "--version", |
| "{placeholder_version}", |
| ], |
| }, |
| "upload_command": { |
| "path": "scripts/upload.sh", |
| "args": ["--namespace", "{upload_namespace}"], |
| }, |
| } |
| props.update(kwargs) |
| return api.properties(jsonpb.ParseDict(props, InputProperties())) |
| |
| yield ( |
| api.buildbucket_util.test("fxt_tests", git_repo=default_remote) |
| + properties( |
| gcs_bucket="foo-artifacts", |
| fxt_options={"image_name": "image-name"}, |
| ) |
| + api.fxt.orchestrate_fxt_tests() |
| ) |
| |
| yield ( |
| api.buildbucket_util.test("jiri_with_mos_upload", git_repo=default_remote) |
| + properties( |
| jiri_manifest="path/to/manifest", |
| download_command={"path": "scripts/download_deps.sh"}, |
| mos_upload_options={ |
| "repo_hostname": "test.fuchsia-update.googleusercontent.com", |
| "gcs_bucket": "discover-cloud.appspot.com", |
| "manifest_path": "path/to/mos/manifest", |
| }, |
| ) |
| ) |