| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Downloads AEMU artifacts from go/ab then uploads to CIPD.""" |
| from PB.recipe_engine.result import RawResult |
| from PB.go.chromium.org.luci.buildbucket.proto import common |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| DEPS = [ |
| "fuchsia/aemu_downloader", |
| "fuchsia/buildbucket_util", |
| "fuchsia/status_check", |
| "recipe_engine/archive", |
| "recipe_engine/cipd", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| CIPD_PKG_ROOT = "fuchsia/third_party/android/aemu" |
| DEFAULT_PKG_TO_SEARCH = "%s/release/linux-amd64" % CIPD_PKG_ROOT |
| |
| ANDROID_TARGET_TO_CIPD_PLATFORM = { |
| "sdk_tools_linux": "linux-amd64", |
| "sdk_tools_mac": "mac-amd64", |
| "aarch64_sdk_tools_linux": "linux-arm64", |
| "aarch64_sdk_tools_mac": "mac-arm64", |
| } |
| |
| |
| def RunSteps(api): |
| """Checks latest AEMU artifact version and upload new version if not latest. |
| |
| This recipe does the following |
| 1. Check the latest Android BuildID (bid) from go/ab that contains all targets. |
| 2. Search CIPD packages based on tag bid:<bid> from (1). |
| 3. If CIPD search returns empty, download all artifacts from go/ab, and upload to CIPD using |
| the mapping: |
| |
| sdk_tools_linux/sdk-repo-linux-emulator-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/release/linux-amd64 |
| |
| sdk_tools_linux/sdk-repo-linux-emulator-full-debug<bid>.zip |
| -> /fuchsia/third_party/android/aemu/full-debug/linux-amd64 |
| |
| sdk_tools_mac/sdk-repo-darwin-emulator-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/release/mac-amd64 |
| |
| sdk_tools_mac/sdk-repo-darwin-emulator-full-debug-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/full-debug/mac-amd64 |
| |
| aarch64_sdk_tools_linux/sdk-repo-linux_aarch64-emulator-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/release/linux-arm64 |
| |
| aarch64_sdk_tools_linux/sdk-repo-linux_aarch64-emulator-full-debug-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/full-debug/linux-arm64 |
| |
| aarch64_sdk_tools_mac/sdk-repo-darwin_aarch64-emulator-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/release/mac-arm64 |
| |
| aarch64_sdk_tools_mac/sdk-repo-darwin_aarch64-emulator-full-debug-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/full-debug/mac-arm64 |
| """ |
| with api.step.nest("check latest versions"): |
| step = api.aemu_downloader.get_latest_build_id( |
| step_name="get latest bid from Android build", |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text("1234567"), |
| ) |
| bid = step.stdout.strip() |
| error_msg = None |
| if not bid: |
| error_msg = "did not find any build id to download" |
| elif not bid.isdigit(): |
| error_msg = "got invalid bid: %s, expect a number" % bid |
| if error_msg: |
| step.presentation.status = api.step.EXCEPTION |
| raise api.step.InfraFailure(error_msg) |
| |
| instances = api.cipd.search( |
| package_name=DEFAULT_PKG_TO_SEARCH, |
| tag="bid:%s" % bid, |
| test_instances=None, |
| ) |
| do_download = not instances |
| |
| if do_download: |
| with api.step.nest("download AEMU artifacts and upload to CIPD"): |
| download_root = api.path["cleanup"].join(bid) |
| api.file.ensure_directory("ensure download dir", download_root) |
| api.aemu_downloader.download( |
| step_name="download bid: %s" % bid, |
| android_build_id=bid, |
| output_root=download_root, |
| ) |
| # The downloaded artifacts follow the file directory pattern |
| # [download_root]/target/artifact. An example would be |
| # [download_root]/sdk_tools_linux/sdk-repo-linux-emulator-1234567.zip where |
| # sdk_tools_linux is the target and sdk-repo-linux-emulator-1234567.zip is the artifact |
| for target in api.file.listdir( |
| "list downloaded targets", |
| download_root, |
| test_data=[ |
| "aarch64_sdk_tools_linux", |
| "aarch64_sdk_tools_mac", |
| "sdk_tools_linux", |
| "sdk_tools_mac", |
| ], |
| ): |
| target_name = api.path.basename(target) |
| cipd_platform = ANDROID_TARGET_TO_CIPD_PLATFORM[target_name] |
| |
| for artifact in api.file.listdir( |
| "list target in %s" % target_name, |
| target, |
| test_data=[ |
| "sdk-repo-linux-emulator-1234567.zip", |
| "sdk-repo-linux-emulator-full-debug-1234567.zip", |
| ], |
| ): |
| artifact_filename = api.path.basename(artifact) |
| extract_output_root, ext = api.path.splitext(artifact_filename) |
| if ext != ".zip": |
| raise api.step.InfraFailure( |
| "Invalid file %s. Expected a .zip file" % artifact_filename |
| ) |
| extract_output_root = target.join(extract_output_root) |
| api.archive.extract( |
| step_name="unzip %s" % artifact_filename, |
| archive_file=artifact, |
| output=extract_output_root, |
| ) |
| # When we unzip, a subdir named 'emulator' is expected. We want to flatten |
| # the file structure by uploading only the contents under 'emulator' dir. |
| # So its important that we fail fast if this structure changes at android |
| # build side. |
| emulator_dir = extract_output_root.join("emulator") |
| api.path.mock_add_directory(emulator_dir) |
| assert api.path.isdir(emulator_dir) |
| if "full-debug" in artifact_filename: |
| pkg_subdir = "full-debug" |
| else: |
| pkg_subdir = "release" |
| cipd_pkg_dest = "%s/%s/%s" % ( |
| CIPD_PKG_ROOT, |
| pkg_subdir, |
| cipd_platform, |
| ) |
| if not api.buildbucket_util.is_tryjob: |
| pkg_def = api.cipd.PackageDefinition( |
| package_name=cipd_pkg_dest, |
| package_root=emulator_dir, |
| install_mode="copy", |
| ) |
| pkg_def.add_dir(emulator_dir) |
| api.cipd.create_from_pkg( |
| pkg_def=pkg_def, |
| refs=["latest"], |
| tags={"bid": bid}, |
| ) |
| return RawResult( |
| status=common.SUCCESS, |
| summary_markdown="Uploaded bid:%s" % bid, |
| ) |
| |
| |
| def GenTests(api): |
| yield ( |
| api.status_check.test("aemu_downloader_do_download") |
| + api.step_data( |
| "check latest versions.cipd search " |
| + DEFAULT_PKG_TO_SEARCH |
| + " bid:1234567", |
| api.cipd.example_search(DEFAULT_PKG_TO_SEARCH, []), |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test( |
| "aemu_downloader_invalid_download", status="infra_failure" |
| ) |
| + api.step_data( |
| "check latest versions.cipd search " |
| + DEFAULT_PKG_TO_SEARCH |
| + " bid:1234567", |
| api.cipd.example_search(DEFAULT_PKG_TO_SEARCH, []), |
| ) |
| + api.step_data( |
| "download AEMU artifacts and upload to CIPD.list target in aarch64_sdk_tools_linux", |
| api.file.listdir( |
| ["sdk-repo-linux-emulator-1234567.zip", "invalid_file.json"] |
| ), |
| ) |
| ) |
| |
| yield ( |
| api.status_check.test("aemu_downloader_no_download") |
| + api.step_data( |
| "check latest versions.cipd search " |
| + DEFAULT_PKG_TO_SEARCH |
| + " bid:1234567", |
| api.cipd.example_search(DEFAULT_PKG_TO_SEARCH, ["bid:1234567"]), |
| ) |
| ) |
| yield ( |
| api.status_check.test("aemu_downloader_invalid_bid", status="infra_failure") |
| + api.step_data( |
| "check latest versions.get latest bid from Android build", |
| stdout=api.raw_io.output_text("abcd1234"), |
| ) |
| ) |
| yield ( |
| api.status_check.test("aemu_downloader_failed_bid", status="failure") |
| + api.step_data( |
| "check latest versions.get latest bid from Android build", |
| retcode=1, |
| stdout=api.raw_io.output_text(""), |
| ) |
| ) |
| yield ( |
| api.status_check.test("aemu_downloader_empty_bid", status="infra_failure") |
| + api.step_data( |
| "check latest versions.get latest bid from Android build", |
| stdout=api.raw_io.output_text(""), |
| ) |
| ) |
