| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """ sdk.py - Builds, tests, and uploads SDKs and companion images. |
| |
| # Execution overview |
| |
| ## Trigger companion image subbuilds |
| |
| Optionally trigger companion images for go/fuchsia-sdk-companion-images. |
| |
| ## Trigger core or Bazel SDK subbuild & collect |
| |
| Trigger a subbuild to build the core or Bazel SDK, which will upload to CAS upon |
| a successful completion. Collect from CAS. |
| |
| ## Generate, test, & upload GN SDK |
| |
| TODO(fxbug.dev/130226): Delete. |
| See https://fuchsia.dev/fuchsia-src/development/sdk/gn for more info. |
| |
| ## Collect companion image subbuilds & upload |
| |
| Collect any companion images, upload PBv2 metadata, and optionally upload the |
| companion images to GCS. |
| |
| ## Upload core or Bazel SDK |
| |
| Upload the core or Bazel SDK to GCS, CAS, and/or CIPD with the appropriate |
| metadata. This signals availability for downstream use, including any uploaded |
| companion images from the previous step. |
| |
| ## Orchestrate external presubmit and/or FXT tests |
| |
| Optionally orchestrate tests against external presubmit(s) to test that the new |
| SDK (and possibly companion images) are compatible. |
| """ |
| |
| import datetime |
| |
| from google.protobuf import json_format |
| from google.protobuf import struct_pb2 |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2 |
| from PB.recipe_engine.result import RawResult |
| from PB.recipes.fuchsia.sdk import InputProperties |
| from PB.recipes.fuchsia.fuchsia.build import InputProperties as SubbuildProperties |
| |
| DEPS = [ |
| "fuchsia/build", |
| "fuchsia/buildbucket_util", |
| "fuchsia/bundle_fetcher", |
| "fuchsia/cas_util", |
| "fuchsia/checkout", |
| "fuchsia/cipd_util", |
| "fuchsia/fxt", |
| "fuchsia/git", |
| "fuchsia/gsutil", |
| "fuchsia/jiri", |
| "fuchsia/presubmit_util", |
| "fuchsia/release", |
| "fuchsia/sso", |
| "fuchsia/subbuild", |
| "recipe_engine/archive", |
| "recipe_engine/buildbucket", |
| "recipe_engine/context", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/platform", |
| "recipe_engine/properties", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| # The name of the output file from bundle_fetcher describing the product bundles |
| # which should be uploaded with the SDK. |
| PRODUCT_BUNDLES_JSON = "product_bundles.json" |
| |
| # These represent the locations of the CAS digest for the SDK archives and Bazel |
| # SDK archives in the output of fuchsia/build.py. Needs to be kept in sync with |
| # fuchsia/build.py. |
| SDK_ARCHIVE_OUTPUT_KEY = "sdk_archive_cas_digest" |
| BAZEL_SDK_OUTPUT_KEY = "bazel_sdk_cas_digest" |
| # This represents a map of tool name to CAS digest for the tools listed in the |
| # tool_overrides property and passed to the fuchsia/build.py recipe through the |
| # cas_tools property. Needs to be kept in sync with fuchsia/build.py. |
| CAS_TOOLS_OUTPUT_KEY = "tools_cas_digests" |
| |
| PROPERTIES = InputProperties |
| |
| |
| def RunSteps(api, props): |
| sdk_id = api.buildbucket_util.id |
| checkout = api.checkout.fuchsia_with_options( |
| project=props.project, |
| manifest=props.manifest, |
| remote=props.remote, |
| allow_skipping_patch=props.allow_skipping_patch, |
| ) |
| |
| integration_root = checkout.project_path(props.project) |
| # Resolve the incoming release version and update refs. |
| update_refs = [] |
| if props.is_release_version: |
| release_version = api.release.get_current_release_version(integration_root) |
| assert release_version, "failed to resolve release version" |
| # Override SDK ID with release version. |
| sdk_id = str(release_version) |
| update_ref = resolve_ref_to_update( |
| api=api, |
| ref_settings=props.ref_settings, |
| remote=props.remote, |
| integration_root=integration_root, |
| release_version=release_version, |
| ) |
| if update_ref: |
| update_refs.append(update_ref) |
| |
| cas_tools = [] |
| if props.sdk_override_configs.tool_overrides: |
| for tool in props.sdk_override_configs.tool_overrides: |
| cas_tools.append(tool.tool_name) |
| extra_properties = json_format.MessageToDict( |
| SubbuildProperties(sdk_id=sdk_id, cas_tools=cas_tools), |
| preserving_proto_field_name=True, |
| ) |
| |
| companion_image_builds = {} |
| if props.companion_images: |
| with api.step.nest("launch companion image subbuilds") as presentation: |
| companion_image_builds = api.subbuild.launch( |
| builder_names=[i.builder for i in props.companion_images], |
| presentation=presentation, |
| # TODO(olivernewman): Swarming currently breaks if many builds |
| # are launched simultaneously with the same |
| # swarming_parent_run_id. Set the swarming_parent_run_id param |
| # here after that bug is fixed. |
| set_swarming_parent_run_id=False, |
| extra_properties=extra_properties, |
| ) |
| |
| with api.step.nest("SDK build") as presentation: |
| builder_name = f"{api.buildbucket.build.builder.builder}-subbuild" |
| builds = api.subbuild.launch( |
| builder_names=[builder_name], |
| presentation=presentation, |
| extra_properties=extra_properties, |
| ) |
| build_id = builds[builder_name].build_id |
| builds = api.subbuild.collect([build_id]) |
| sdk_build = builds[build_id].build_proto |
| api.buildbucket_util.display_builds( |
| step_name="check SDK build status", |
| builds=[sdk_build], |
| raise_on_failure=True, |
| ) |
| # Copy various output properties from the child build to the parent so |
| # that tools don't need to look up the child build in order to get these |
| # values. |
| output_props = json_format.MessageToDict(sdk_build.output.properties) |
| for prop in ( |
| api.build.FINT_PARAMS_PATH_PROPERTY, |
| api.checkout.CHECKOUT_INFO_PROPERTY, |
| api.checkout.CACHED_REVISION_PROPERTY, |
| ): |
| if prop in output_props: |
| presentation.properties[prop] = output_props[prop] |
| |
| core_sdk_archive_path = run_core_sdk_steps(api, sdk_build) |
| |
| sdk_id_gcs_prefix = f"development/{sdk_id}" |
| sdk_gcs_prefix = f"{sdk_id_gcs_prefix}/sdk" |
| sdk_gcs_path = f"{sdk_gcs_prefix}/{api.platform.name}-amd64/{props.sdk_name}.tar.gz" |
| cipd_metadata = [ |
| ("jiri_snapshot", sdk_id), |
| ("version", sdk_id), |
| ] |
| revision = api.buildbucket.build.input.gitiles_commit.id |
| # TODO(fxbug.dev/130226): Delete. |
| if core_sdk_archive_path and not props.skip_gn_sdk: |
| run_gn_sdk_steps( |
| api, |
| props, |
| checkout, |
| core_sdk_archive_path, |
| sdk_gcs_prefix, |
| sdk_id, |
| revision, |
| update_refs, |
| cipd_metadata, |
| ) |
| |
| bazel_sdk_path = run_bazel_sdk_steps(api, sdk_build) |
| |
| assert ( |
| core_sdk_archive_path or bazel_sdk_path |
| ), "neither a core nor Bazel SDK were built" |
| |
| tools_digests = None |
| if props.sdk_override_configs.tool_overrides: |
| tools_digests = api.subbuild.get_property(sdk_build, CAS_TOOLS_OUTPUT_KEY) |
| |
| pbv2_path = None |
| package_overrides = {} |
| |
| if props.companion_images: |
| with api.step.nest("collect companion image subbuilds"): |
| companion_image_builds = api.subbuild.collect( |
| build_ids=[b.build_id for b in companion_image_builds.values()], |
| ) |
| api.buildbucket_util.display_builds( |
| step_name="check companion image subbuilds", |
| builds=[b.build_proto for b in companion_image_builds.values()], |
| raise_on_failure=True, |
| ) |
| pbv2_path = run_companion_image_steps( |
| api, |
| props, |
| companion_image_builds, |
| sdk_id, |
| package_overrides, |
| ) |
| |
| run_upload_steps( |
| api, |
| props, |
| core_sdk_archive_path, |
| bazel_sdk_path, |
| tools_digests, |
| pbv2_path, |
| checkout, |
| sdk_id, |
| revision, |
| sdk_gcs_path, |
| sdk_id_gcs_prefix, |
| cipd_metadata, |
| update_refs, |
| package_overrides, |
| ) |
| |
| if package_overrides: |
| with api.step.nest("set cas_digests output property") as presentation: |
| presentation.properties["cas_digests"] = package_overrides |
| |
| if ( |
| props.external_testing.gerrit_host |
| or props.sdk_override_configs.use_gclient_variables |
| ): |
| with api.step.nest("run external tests"): |
| run_external_tests( |
| api, |
| sdk_id=sdk_id, |
| bucket=props.gcs_bucket, |
| sdk_gcs_prefix=sdk_gcs_prefix, |
| sdk_gcs_path=sdk_gcs_path, |
| package_overrides=package_overrides, |
| sdk_override_configs=props.sdk_override_configs, |
| external_testing=props.external_testing, |
| ) |
| |
| run_fxt_tests = any( |
| [ |
| props.fxt_options.tap_projects, |
| props.fxt_options.guitar_config, |
| props.fxt_options.piper_presubmit, |
| ] |
| ) |
| |
| if run_fxt_tests: |
| api.fxt.orchestrate_fxt_tests( |
| bucket=props.gcs_bucket, |
| namespace=sdk_id, |
| options=props.fxt_options, |
| ) |
| |
| |
| def fetch_sdk_from_subbuild(api, build_proto, output_key, artifact_type): |
| with api.step.nest(f"fetch {artifact_type} artifacts"): |
| try: |
| digest = api.subbuild.get_property(build_proto, output_key) |
| except api.step.InfraFailure: |
| return None |
| output_path = api.path.cleanup_dir / output_key |
| api.cas_util.download(digest=digest, output_dir=output_path) |
| return output_path |
| |
| |
| def run_core_sdk_steps(api, sdk_build): |
| core_sdk_path = fetch_sdk_from_subbuild( |
| api, |
| sdk_build, |
| SDK_ARCHIVE_OUTPUT_KEY, |
| "sdk", |
| ) |
| if not core_sdk_path: |
| return None |
| |
| core_sdk_archive_paths = api.file.glob_paths( |
| "get core SDK path", |
| core_sdk_path, |
| "*.tar.gz", |
| test_data=["sdk.tar.gz"], |
| ) |
| assert len(core_sdk_archive_paths) == 1 |
| return core_sdk_archive_paths[0] |
| |
| |
| # TODO(fxbug.dev/130226): Delete. |
| def run_gn_sdk_steps( |
| api, |
| props, |
| checkout, |
| core_sdk_archive_path, |
| sdk_gcs_prefix, |
| sdk_id, |
| revision, |
| update_refs, |
| cipd_metadata, |
| ): |
| with api.step.nest("gen") as presentation: |
| # Run GN to get access to tool_paths.json for looking up paths to |
| # prebuilt tools. |
| gn_results = api.build.gen( |
| checkout=checkout, |
| fint_params_path=props.fint_params_path, |
| presentation=presentation, |
| ) |
| |
| # Create a `python` executable that will be available in $PATH as an |
| # alias for `python3`, since there's no `python` executable in the |
| # checkout. |
| python3 = gn_results.tool("python3") |
| python_dir = api.path.mkdtemp("python") |
| api.file.symlink("symlink python3 to python", python3, python_dir / "python") |
| # As per RFC-0129, "Python scripts intended to be directly invoked MUST |
| # contain a shebang which ultimately references the vendored Python". |
| # This applies to all Python GN build scripts, some of which are |
| # triggered in the SDK tests. To unblock Python build script migration |
| # to "fuchsia-vendored-python", we need to provide it as a runtime |
| # executable in this checkout accordingly. |
| api.file.symlink( |
| "symlink python3 to fuchsia-vendored-python", |
| python3, |
| python_dir / "fuchsia-vendored-python", |
| ) |
| |
| def run_python(step_name, cmd, **kwargs): |
| # Override $PATH to make sure that the script only uses the prebuilt |
| # Python from the checkout, and doesn't fall back to whatever is |
| # ambiently installed. |
| path_prefix = [python_dir, api.path.dirname(python3)] |
| with api.context(env_prefixes={"PATH": path_prefix}): |
| api.step(step_name, [python3] + cmd, **kwargs) |
| |
| with api.step.nest("generate gn sdk"): |
| # Generate a GN workspace along with its tests. |
| # These tests are being run for every SDK flavor. |
| generate_gn_path = checkout.root_dir.joinpath( |
| "scripts", "sdk", "gn", "generate.py" |
| ) |
| gn_sdk_dir = api.path.cleanup_dir / "sdk-gn" |
| gn_sdk_archive = api.path.cleanup_dir / "gn.tar.gz" |
| test_workspace_dir = api.path.cleanup_dir / "tests" |
| |
| run_python( |
| "create gn sdk", |
| [ |
| generate_gn_path, |
| "--archive", |
| core_sdk_archive_path, |
| "--output", |
| gn_sdk_dir, |
| "--output-archive", |
| gn_sdk_archive, |
| "--tests", |
| test_workspace_dir, |
| ], |
| ) |
| with api.step.nest("test gn sdk"): |
| run_python( |
| "run gn tests", |
| [test_workspace_dir / "run.py"], |
| timeout=datetime.timedelta(hours=1), |
| ) |
| |
| with api.step.nest("publish gn sdk"): |
| gcs_path = f"{sdk_gcs_prefix}/{api.platform.name}-amd64/gn.tar.gz" |
| api.gsutil.upload( |
| bucket=props.gcs_bucket, |
| src=gn_sdk_archive, |
| dst=gcs_path, |
| name=f"upload gn fuchsia-sdk {sdk_id}", |
| unauthenticated_url=True, |
| ) |
| if props.is_release_version: |
| api.cipd_util.upload_package( |
| f"{props.cipd_root}/sdk/gn/{api.cipd_util.platform_name}", |
| gn_sdk_dir, |
| search_tag={"git_revision": revision}, |
| repository=None, |
| refs=update_refs, |
| metadata=cipd_metadata, |
| ) |
| |
| |
| def run_bazel_sdk_steps(api, sdk_build): |
| return fetch_sdk_from_subbuild( |
| api, |
| sdk_build, |
| BAZEL_SDK_OUTPUT_KEY, |
| "bazel SDK", |
| ) |
| |
| |
| def run_companion_image_steps( |
| api, |
| props, |
| companion_image_builds, |
| sdk_id, |
| package_overrides, |
| ): |
| for b in companion_image_builds.values(): |
| try: |
| cas_digests = api.subbuild.get_property( |
| b.build_proto, |
| "cas_digests", |
| ) |
| except api.step.InfraFailure: |
| continue |
| for package, digest in cas_digests.items(): |
| assert ( |
| package not in package_overrides |
| ), f"duplicate overridden package {package}" |
| package_overrides[package] = digest |
| |
| artifact_gcs_buckets = list( |
| set( |
| api.subbuild.get_property(b.build_proto, "artifact_gcs_bucket") |
| for b in companion_image_builds.values() |
| ) |
| ) |
| assert len(artifact_gcs_buckets) == 1, artifact_gcs_buckets |
| artifact_gcs_bucket = artifact_gcs_buckets[0] |
| |
| with api.step.nest("resolve PB metadata"): |
| pbv2_dir = api.path.mkdtemp("pbv2") |
| api.bundle_fetcher.product_list( |
| build_ids=[b.build_id for b in companion_image_builds.values()], |
| gcs_bucket=artifact_gcs_bucket, |
| out_dir=pbv2_dir, |
| ) |
| pbv2_path = pbv2_dir / PRODUCT_BUNDLES_JSON |
| |
| if props.relocate_image_archives: |
| # Relocate the image archives into a well-known layout based on the |
| # SDK ID. Requires that the (deprecated) archives are published to GCS. |
| # TODO(fxbug.dev/127210): Remove once all users are migrated to PBv2. |
| with api.step.nest("relocate companion image archives"): |
| for b in companion_image_builds.values(): |
| image_name = next( |
| image.name |
| for image in props.companion_images |
| if image.builder == b.build_proto.builder.builder |
| ) |
| src_bucket = api.subbuild.get_property(b.build_proto, "gcs_bucket") |
| src_bucket = src_bucket.strip('"') |
| api.gsutil.copy( |
| src_bucket=src_bucket, |
| src=f"builds/{b.build_proto.id}/build-archive.tgz", |
| dst_bucket=props.gcs_bucket, |
| dst=f"development/{sdk_id}/images/{image_name}.tgz", |
| link_name=f"{image_name} build-archive", |
| ) |
| |
| return pbv2_path |
| |
| |
| def run_upload_steps( |
| api, |
| props, |
| core_sdk_archive_path, |
| bazel_sdk_path, |
| tools_digests, |
| pbv2_path, |
| checkout, |
| sdk_id, |
| revision, |
| sdk_gcs_path, |
| sdk_id_gcs_prefix, |
| cipd_metadata, |
| update_refs, |
| package_overrides, |
| ): |
| if core_sdk_archive_path: |
| with api.step.nest("upload core SDK"): |
| api.gsutil.upload( |
| bucket=props.gcs_bucket, |
| src=core_sdk_archive_path, |
| dst=sdk_gcs_path, |
| link_name="core SDK archive", |
| unauthenticated_url=True, |
| ) |
| |
| # Note that this will upload the snapshot to a location different from the |
| # path that api.fuchsia copied it to. This uses a path based on the hash of |
| # the SDK artifact, not based on the hash of the snapshot itself. Clients |
| # can use this to find the snapshot used to build a specific SDK artifact. |
| snapshot_file = api.path.cleanup_dir / "jiri.snapshot" |
| with api.context(cwd=checkout.root_dir): |
| api.jiri.snapshot(snapshot_file) |
| api.gsutil.upload( |
| bucket="fuchsia-snapshots", |
| src=snapshot_file, |
| dst=sdk_id, |
| link_name="jiri.snapshot", |
| name="upload jiri.snapshot", |
| ) |
| |
| for update_ref in update_refs: |
| upper_update_ref = update_ref.upper() |
| api.gsutil.upload( |
| bucket=props.gcs_bucket, |
| src=api.raw_io.input(sdk_id), |
| dst=f"development/{upper_update_ref}_{api.platform.name.upper()}", |
| link_name=upper_update_ref, |
| name=f"upload {update_ref} sdk_id", |
| ) |
| |
| # Extract archive to a directory for CIPD processing. |
| cipd_staging_dir = api.path.cleanup_dir / props.sdk_name |
| api.file.ensure_directory("create SDK dir", cipd_staging_dir) |
| api.archive.extract( |
| step_name="unpack SDK archive", |
| archive_file=core_sdk_archive_path, |
| output=cipd_staging_dir, |
| ) |
| cipd_pkg = ( |
| f"{props.cipd_root}/sdk/{props.sdk_name}/{api.cipd_util.platform_name}" |
| ) |
| if props.is_release_version: |
| api.cipd_util.upload_package( |
| cipd_pkg, |
| cipd_staging_dir, |
| search_tag={"git_revision": revision}, |
| repository=None, |
| refs=update_refs, |
| metadata=cipd_metadata, |
| # The default timeout (5 mins) is sometime not sufficient. See |
| # crbug.com/1404130 for more details. |
| verification_timeout="10m", |
| ) |
| else: |
| package_overrides[cipd_pkg] = api.cas_util.upload(cipd_staging_dir) |
| if bazel_sdk_path: |
| with api.step.nest("upload bazel SDK"): |
| cipd_pkg = f"{props.cipd_root}/sdk/core/fuchsia-bazel-rules/{api.cipd_util.platform_name}" |
| if props.is_release_version: |
| api.cipd_util.upload_package( |
| cipd_pkg, |
| bazel_sdk_path, |
| search_tag={"git_revision": revision}, |
| repository=None, |
| refs=update_refs, |
| metadata=cipd_metadata, |
| ) |
| else: |
| package_overrides[cipd_pkg] = api.cas_util.upload(bazel_sdk_path) |
| |
| if tools_digests: |
| for tool in props.sdk_override_configs.tool_overrides: |
| package_overrides[ |
| tool.cipd_pkg_name.replace("${platform}", api.cipd_util.platform_name) |
| ] = tools_digests[tool.tool_name] |
| |
| if pbv2_path: |
| with api.step.nest("upload PBv2 metadata"): |
| pbv2_gcs_path = f"{sdk_id_gcs_prefix}/{PRODUCT_BUNDLES_JSON}" |
| api.gsutil.upload( |
| bucket=props.gcs_bucket, |
| src=pbv2_path, |
| dst=pbv2_gcs_path, |
| name=f"upload {PRODUCT_BUNDLES_JSON} v2", |
| unauthenticated_url=True, |
| ) |
| pbv2_staging_dir = api.path.mkdtemp("pbv2_staging") |
| api.file.copy("stage manifest", pbv2_path, pbv2_staging_dir) |
| pbv2_cipd_pkg = f"{props.cipd_root}/development/product_bundles/v2" |
| if props.is_release_version: |
| api.cipd_util.upload_package( |
| pbv2_cipd_pkg, |
| pbv2_staging_dir, |
| search_tag={"git_revision": revision}, |
| repository=None, |
| refs=update_refs, |
| metadata=cipd_metadata, |
| ) |
| else: |
| package_overrides[pbv2_cipd_pkg] = api.cas_util.upload(pbv2_staging_dir) |
| |
| |
| def resolve_ref_to_update(api, ref_settings, remote, integration_root, release_version): |
| """Resolve which ref to update. |
| |
| Ref settings order is respected i.e. the first branch which contains |
| the input release version will be used. If the release version is |
| not reachable on any of the given branches, return None. |
| """ |
| for settings in ref_settings: |
| branch_head = api.git.get_remote_branch_head( |
| api.sso.sso_to_https(remote), settings.branch |
| ) |
| with api.context(cwd=integration_root): |
| api.git.fetch("origin", refspec=branch_head) |
| if release_version in api.release.get_release_versions( |
| ref=branch_head, repo_path=integration_root |
| ): |
| return settings.update_ref |
| return None # pragma: no cover |
| |
| |
| def run_external_tests( |
| api, |
| sdk_id, |
| bucket, |
| sdk_gcs_prefix, |
| sdk_gcs_path, |
| package_overrides, |
| sdk_override_configs, |
| external_testing, |
| ): |
| """Dry-run SDK in an external Buildbucket-based presubmit.""" |
| file_edits = None |
| final_package_overrides = None |
| gclient_variables = None |
| |
| if sdk_override_configs.use_package_overrides: |
| final_package_overrides = package_overrides |
| elif sdk_override_configs.use_gclient_variables: |
| gclient_variables = { |
| "download_fuchsia_sdk": True, |
| "fuchsia_sdk_path": sdk_gcs_path, |
| } |
| elif sdk_override_configs.override_config_file: |
| file_edits = [ |
| ( |
| sdk_override_configs.override_config_file, |
| f"gs://{bucket}/{sdk_gcs_prefix}", |
| ) |
| ] |
| else: # pragma: no cover |
| return RawResult( |
| summary_markdown="no SDK override mechanism specified", |
| status=common_pb2.INFRA_FAILURE, |
| ) |
| |
| api.presubmit_util.orchestrate( |
| options=external_testing, |
| cl_subject=f"[test] Dryrun SDK version {sdk_id}", |
| file_edits=file_edits, |
| package_overrides=final_package_overrides, |
| gclient_variables=gclient_variables, |
| ) |
| |
| |
| def GenTests(api): |
| def properties(**kwargs): |
| props = { |
| "project": "integration", |
| "manifest": "flower", |
| "remote": "https://fuchsia.googlesource.com/integration", |
| "gcs_bucket": "fuchsia", |
| "sdk_name": "core", |
| "cipd_root": "fuchsia", |
| "fint_params_path": "fint_params/sdk.textproto", |
| } |
| props.update(kwargs) |
| return api.properties(**props) |
| |
| def sdk_subbuild(output_key, cas_tools=()): |
| b = api.subbuild.ci_build_message( |
| build_id=123, |
| builder="builder-subbuild", |
| output_props={ |
| "checkout_info": {"manifest": "foo"}, |
| }, |
| status="SUCCESS", |
| ) |
| b.output.properties[output_key] = "cas-digest" |
| tools_digests = {} |
| for i, tool in enumerate(cas_tools): |
| tools_digests[tool] = f"cas-digest/{i}" |
| if tools_digests: |
| b.output.properties[CAS_TOOLS_OUTPUT_KEY] = tools_digests |
| return api.subbuild.child_build_steps( |
| builds=[b], |
| launch_step="SDK build", |
| collect_step="SDK build", |
| ) |
| |
| image_build = api.subbuild.ci_build_message( |
| builder="###SDK_IMAGE_BUILDER###", |
| status="SUCCESS", |
| build_id=123456789, |
| ) |
| image_build.output.properties["artifact_gcs_bucket"] = "fuchsia-artifacts" |
| image_build.output.properties["gcs_bucket"] = "###BUCKET###" |
| image_build.output.properties["cas_digests"] = {"fuchsia/tools/foo": "abc123/50"} |
| other_image_build = api.subbuild.ci_build_message( |
| builder="###OTHER_SDK_IMAGE_BUILDER###", |
| status="SUCCESS", |
| build_id=9876543210, |
| ) |
| other_image_build.output.properties["artifact_gcs_bucket"] = "fuchsia-artifacts" |
| other_image_build.output.properties["gcs_bucket"] = "###BUCKET###" |
| |
| ci_image_builds = api.subbuild.child_build_steps( |
| builds=[image_build, other_image_build], |
| launch_step="launch companion image subbuilds", |
| collect_step="collect companion image subbuilds", |
| ) |
| |
| ref_settings = [ |
| {"branch": "refs/heads/main", "update_ref": "latest"}, |
| ] |
| release_versions = api.step_data( |
| "get release versions on h3ll0", |
| api.raw_io.stream_output_text( |
| "\n".join( |
| [ |
| "releases/0.20191019.0.1", |
| "releases/0.20191018.0.1", |
| ] |
| ) |
| ), |
| ) |
| |
| companion_images = [ |
| { |
| "name": "###SDK_IMAGE###", |
| "builder": "###SDK_IMAGE_BUILDER###", |
| }, |
| { |
| "name": "###OTHER_SDK_IMAGE###", |
| "builder": "###OTHER_SDK_IMAGE_BUILDER###", |
| }, |
| ] |
| yield ( |
| api.buildbucket_util.test("fxt_tests") |
| + properties( |
| fxt_options=api.fxt.Options( |
| sdk_mode=True, |
| tap_projects=["foo"], |
| guitar_config=json_format.ParseDict( |
| {"projectfoo": "bar"}, struct_pb2.Struct() |
| ), |
| piper_presubmit=True, |
| use_staging_host=False, |
| timeout_secs=60 * 60, |
| ), |
| ) |
| + sdk_subbuild(SDK_ARCHIVE_OUTPUT_KEY) |
| + api.fxt.orchestrate_fxt_tests(piper_presubmit=True) |
| ) |
| yield ( |
| api.buildbucket_util.test("external_testing") |
| + properties( |
| ref_settings=ref_settings, |
| sdk_override_configs=InputProperties.SDKOverrideConfigs( |
| override_config_file="override_config", |
| ), |
| external_testing=api.presubmit_util.Options( |
| gerrit_host="foo-review.googlesource.com", |
| gerrit_project="repo", |
| ), |
| ) |
| + sdk_subbuild(SDK_ARCHIVE_OUTPUT_KEY) |
| + api.presubmit_util.create_cl(nesting="run external tests") |
| + api.presubmit_util.wait_for_cq(nesting="run external tests") |
| ) |
| yield ( |
| api.buildbucket_util.test("package_overrides") |
| + properties( |
| ref_settings=ref_settings, |
| sdk_override_configs=InputProperties.SDKOverrideConfigs( |
| use_package_overrides=True, |
| tool_overrides=[ |
| { |
| "tool_name": "tool1", |
| "cipd_pkg_name": "fuchsia/tool1/${platform}", |
| }, |
| { |
| "tool_name": "tool2", |
| "cipd_pkg_name": "fuchsia/tool2/${platform}", |
| }, |
| ], |
| ), |
| companion_images=companion_images, |
| external_testing=api.presubmit_util.Options( |
| gerrit_host="foo-review.googlesource.com", |
| gerrit_project="repo", |
| ), |
| ) |
| + sdk_subbuild(BAZEL_SDK_OUTPUT_KEY, cas_tools=["tool1", "tool2"]) |
| + ci_image_builds |
| + api.presubmit_util.create_cl(nesting="run external tests") |
| + api.presubmit_util.wait_for_cq(nesting="run external tests") |
| ) |
| tryjob_msgs = [ |
| api.buildbucket.try_build_message( |
| project="fuchsia", |
| bucket="try", |
| builder="bar.x64-debug", |
| ), |
| ] |
| yield ( |
| api.buildbucket_util.test("gclient_variables") |
| + properties( |
| ref_settings=ref_settings, |
| sdk_override_configs=InputProperties.SDKOverrideConfigs( |
| use_gclient_variables=True, |
| ), |
| external_testing=api.presubmit_util.Options( |
| tryjobs=["foo/try/a_builder"], |
| ), |
| companion_images=companion_images, |
| ) |
| + sdk_subbuild(SDK_ARCHIVE_OUTPUT_KEY) |
| + ci_image_builds |
| + api.presubmit_util.trigger_tryjobs( |
| nesting="run external tests", tryjob_msgs=tryjob_msgs |
| ) |
| + api.presubmit_util.collect_tryjobs( |
| nesting="run external tests", tryjob_msgs=tryjob_msgs |
| ) |
| ) |
| yield ( |
| api.buildbucket_util.test( |
| "release_with_companion_images", |
| git_ref="refs/heads/releases/foo", |
| ) |
| + properties( |
| is_release_version=True, |
| companion_images=companion_images, |
| relocate_image_archives=True, |
| ref_settings=ref_settings, |
| ) |
| + sdk_subbuild(SDK_ARCHIVE_OUTPUT_KEY) |
| + ci_image_builds |
| + api.release.ref_to_release_version("releases/0.20191018.0.1") |
| + release_versions |
| ) |
| yield ( |
| api.buildbucket_util.test( |
| "bazel_release", |
| git_ref="refs/heads/releases/foo", |
| ) |
| + properties( |
| is_release_version=True, |
| ref_settings=ref_settings, |
| ) |
| + sdk_subbuild(BAZEL_SDK_OUTPUT_KEY) |
| + api.release.ref_to_release_version("releases/0.20191018.0.1") |
| + release_versions |
| ) |
