| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Downloads AEMU artifacts from go/ab then uploads to CIPD.""" |
| # TODO(fxbug.dev/116835): Write new recipe that makes this workflow generic, |
| # migrate existing package |
| from PB.recipe_engine.result import RawResult |
| from PB.go.chromium.org.luci.buildbucket.proto import common |
| |
| DEPS = [ |
| "fuchsia/android_downloader", |
| "fuchsia/buildbucket_util", |
| "recipe_engine/archive", |
| "recipe_engine/cipd", |
| "recipe_engine/file", |
| "recipe_engine/path", |
| "recipe_engine/raw_io", |
| "recipe_engine/step", |
| ] |
| |
| AEMU_CIPD_PKG_ROOT = "fuchsia/third_party/android/aemu" |
| STARNIX_CIPD_PKG_ROOT = "fuchsia_internal/starnix" |
| AEMU_DEFAULT_PKG_TO_SEARCH = f"{AEMU_CIPD_PKG_ROOT}/release/linux-amd64" |
| STARNIX_DEFAULT_PKG_TO_SEARCH = f"{STARNIX_CIPD_PKG_ROOT}/android-image-amd64" |
| AEMU_ANDROID_BRANCH = "aosp-emu-master-dev" |
| STARNIX_ANDROID_BRANCH = "git_starnix-dev" |
| |
| # The downloaded artifacts follow the file directory pattern |
| # [download_root]/target/artifact. An example would be |
| # [download_root]/emulator-linux_x64_internal/sdk-repo-linux-emulator-1234567.zip where |
| # emulator-linux_x64_internal is the target and sdk-repo-linux-emulator-1234567.zip is the artifact |
| AEMU_TARGET_TO_CIPD_PLATFORM = { |
| "emulator-linux_x64_internal": "linux-amd64", |
| "emulator-mac_x64": "mac-amd64", |
| "emulator-linux_aarch64": "linux-arm64", |
| "emulator-mac_aarch64": "mac-arm64", |
| } |
| |
| AEMU_ARTIFACTS = { |
| "emulator-linux_x64_internal": [ |
| "sdk-repo-linux-emulator-{bid}.zip", |
| ], |
| "emulator-linux_aarch64": [ |
| "sdk-repo-linux_aarch64-emulator-{bid}.zip", |
| ], |
| "emulator-mac_x64": [ |
| "sdk-repo-darwin-emulator-{bid}.zip", |
| ], |
| "emulator-mac_aarch64": [ |
| "sdk-repo-darwin_aarch64-emulator-{bid}.zip", |
| ], |
| } |
| |
| STARNIX_TARGET_TO_CIPD_PLATFORM = { |
| "starnix_x86_64-eng": "android-image-amd64", |
| "starnix_x86_64-eng-symbols": "android-image-debug-symbols-amd64", |
| "starnix_arm64-eng": "android-image-arm64", |
| "starnix_arm64-eng-symbols": "android-image-debug-symbols-arm64", |
| } |
| |
| STARNIX_ARTIFACTS = { |
| "starnix_x86_64-eng": [ |
| "starnix_x86_64-img-{bid}.zip", |
| "bin/adb", |
| "bin/aidl", |
| "bin/aidl_rust_glue.py", |
| "lib64/libc++.so", |
| "art-run-tst-target-data-merged.zip", |
| "starnix_testcases.img", |
| "build-id.zip", |
| ], |
| "starnix_arm64-eng": [ |
| "starnix_arm64-img-{bid}.zip", |
| "bin/adb", |
| "bin/aidl", |
| "bin/aidl_rust_glue.py", |
| "lib64/libc++.so", |
| "art-run-tst-target-data-merged.zip", |
| "starnix_testcases.img", |
| "build-id.zip", |
| ], |
| } |
| |
| |
| def get_bid(api, starnix): |
| """Gets the latest version of the target""" |
| with api.step.nest(f"check latest versions {'starnix' if starnix else 'AEMU'}"): |
| step = api.android_downloader.get_latest_build_id( |
| step_name=f"get latest bid from Android build for {'starnix' if starnix else 'AEMU'}", |
| artifacts=STARNIX_ARTIFACTS if starnix else AEMU_ARTIFACTS, |
| branch=STARNIX_ANDROID_BRANCH if starnix else AEMU_ANDROID_BRANCH, |
| stdout=api.raw_io.output_text(), |
| step_test_data=lambda: api.raw_io.test_api.stream_output_text("1234567"), |
| ok_ret="any" if starnix else (0,), |
| ) |
| bid = step.stdout.strip() |
| |
| if starnix: |
| return bid |
| |
| error_msg = None |
| if not bid: |
| error_msg = "did not find any build id to download" |
| elif not bid.isdigit(): |
| error_msg = f"got invalid bid: {bid}, expect a number" |
| if error_msg: |
| step.presentation.status = api.step.EXCEPTION |
| raise api.step.InfraFailure(error_msg) |
| |
| return bid |
| |
| |
| def download_aemu(api, bid, platforms): |
| with api.step.nest("download AEMU artifacts and upload to CIPD"): |
| download_root = api.path["cleanup"].join(bid) |
| api.file.ensure_directory("ensure download dir", download_root) |
| api.android_downloader.download( |
| step_name=f"download bid: {bid}", |
| android_build_id=bid, |
| artifacts=AEMU_ARTIFACTS, |
| branch=AEMU_ANDROID_BRANCH, |
| output_root=download_root, |
| ) |
| |
| for target in api.file.listdir( |
| "list downloaded targets", |
| download_root, |
| test_data=AEMU_TARGET_TO_CIPD_PLATFORM.keys(), |
| ): |
| target_name = api.path.basename(target) |
| cipd_platform = platforms[target_name] |
| |
| for artifact in api.file.listdir( |
| f"list target in {target_name}", |
| target, |
| test_data=[ |
| "sdk-repo-linux-emulator-1234567.zip", |
| ], |
| ): |
| artifact_filename = api.path.basename(artifact) |
| extract_output_root, ext = api.path.splitext(artifact_filename) |
| if ext != ".zip": |
| raise api.step.InfraFailure( |
| f"Invalid file {artifact_filename}. Expected a .zip file" |
| ) |
| extract_output_root = target.join(extract_output_root) |
| api.archive.extract( |
| step_name=f"unzip {artifact_filename}", |
| archive_file=artifact, |
| output=extract_output_root, |
| ) |
| # When we unzip, a subdir named 'emulator' is expected. We want to flatten |
| # the file structure by uploading only the contents under 'emulator' dir. |
| # So its important that we fail fast if this structure changes at android |
| # build side. |
| emulator_dir = extract_output_root.join("emulator") |
| api.path.mock_add_directory(emulator_dir) |
| assert api.path.isdir(emulator_dir) |
| pkg_subdir = "release" |
| cipd_pkg_dest = f"{AEMU_CIPD_PKG_ROOT}/{pkg_subdir}/{cipd_platform}" |
| if not api.buildbucket_util.is_tryjob: |
| pkg_def = api.cipd.PackageDefinition( |
| package_name=cipd_pkg_dest, |
| package_root=emulator_dir, |
| install_mode="copy", |
| ) |
| pkg_def.add_dir(emulator_dir) |
| api.cipd.create_from_pkg( |
| pkg_def=pkg_def, |
| refs=["latest"], |
| tags={"bid": bid}, |
| ) |
| return RawResult( |
| status=common.SUCCESS, |
| summary_markdown=f"Uploaded bid:{bid} for AEMU", |
| ) |
| |
| |
| def download_starnix(api, bid): |
| with api.step.nest("download Starnix artifacts and upload to CIPD"): |
| download_root = api.path["cleanup"].join(bid) |
| api.file.ensure_directory("ensure download dir", download_root) |
| api.android_downloader.download( |
| step_name=f"download starnix bid: {bid}", |
| artifacts=STARNIX_ARTIFACTS, |
| branch=STARNIX_ANDROID_BRANCH, |
| android_build_id=bid, |
| output_root=download_root, |
| ) |
| |
| for target in api.file.listdir( |
| "list downloaded targets", |
| download_root, |
| test_data=STARNIX_TARGET_TO_CIPD_PLATFORM.keys(), |
| ): |
| target_name = api.path.basename(target) |
| cipd_platform = STARNIX_TARGET_TO_CIPD_PLATFORM[target_name] |
| cipd_pkg_dest = f"{STARNIX_CIPD_PKG_ROOT}/{cipd_platform}" |
| |
| image_name = f"starnix_x86_64-img-{bid}.zip" |
| if "arm64" in target_name: |
| image_name = f"starnix_arm64-img-{bid}.zip" |
| image_zip = f"{target}/{image_name}" |
| |
| cipd_symbols_platform = STARNIX_TARGET_TO_CIPD_PLATFORM[ |
| target_name + "-symbols" |
| ] |
| cipd_symbols_pkg_dest = f"{STARNIX_CIPD_PKG_ROOT}/{cipd_symbols_platform}" |
| symbol_zip = target.join("build-id.zip") |
| symbol_dir = api.path["cleanup"].join(f"{bid}-build-id") |
| api.file.ensure_directory("ensure symbol dir", symbol_dir) |
| |
| api.archive.extract( |
| step_name=f"unzip {image_zip} to {target}", |
| archive_file=image_zip, |
| output=target, |
| ) |
| api.file.remove("remove image zip file", image_zip) |
| |
| api.archive.extract( |
| step_name=f"unzip {symbol_zip} to {symbol_dir}", |
| archive_file=symbol_zip, |
| output=symbol_dir, |
| ) |
| api.file.remove("remove symbol zip file", symbol_zip) |
| |
| bin_dir = target.join("bin") |
| for bin_target in api.file.listdir( |
| "list binaries", bin_dir, test_data=["adb"] |
| ): |
| api.step( |
| f"make {api.path.basename(bin_target)} executable", |
| ["chmod", "+x", bin_target], |
| ) |
| |
| if not api.buildbucket_util.is_tryjob: |
| pkg_def = api.cipd.PackageDefinition( |
| package_name=cipd_pkg_dest, |
| package_root=target, |
| install_mode="copy", |
| ) |
| pkg_def.add_dir(target) |
| api.cipd.create_from_pkg( |
| pkg_def=pkg_def, |
| refs=["latest"], |
| tags={"bid": bid}, |
| ) |
| |
| symbols_pkg_def = api.cipd.PackageDefinition( |
| package_name=cipd_symbols_pkg_dest, |
| package_root=symbol_dir, |
| install_mode="copy", |
| ) |
| symbols_pkg_def.add_dir(symbol_dir) |
| api.cipd.create_from_pkg( |
| pkg_def=symbols_pkg_def, |
| refs=["latest"], |
| tags={"bid": bid}, |
| ) |
| |
| return RawResult( |
| status=common.SUCCESS, summary_markdown=f"Uploaded bid:{bid} for Starnix" |
| ) |
| |
| |
| def RunSteps(api): |
| """Checks latest AEMU artifact version and upload new version if not latest. |
| |
| This recipe does the following |
| 1. Check the latest Android BuildID (bid) from go/ab that contains all targets. |
| 2. Search CIPD packages based on tag bid:<bid> from (1). |
| 3. If CIPD search returns empty, download all artifacts from go/ab, and upload to CIPD using |
| the mapping: |
| |
| emulator-linux_x64_internal/sdk-repo-linux-emulator-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/release/linux-amd64 |
| |
| emulator-mac_x64/sdk-repo-darwin-emulator-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/release/mac-amd64 |
| |
| emulator-linux_aarch64/sdk-repo-linux_aarch64-emulator-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/release/linux-arm64 |
| |
| emulator-mac_aarch64/sdk-repo-darwin_aarch64-emulator-<bid>.zip |
| -> /fuchsia/third_party/android/aemu/release/mac-arm64 |
| |
| starnix/* |
| -> /fuchsia/starnix/internal/android-image-amd64/* |
| """ |
| aemu_bid = get_bid(api, False) |
| |
| aemu_instances = api.cipd.search( |
| package_name=AEMU_DEFAULT_PKG_TO_SEARCH, |
| tag=f"bid:{aemu_bid}", |
| test_instances=None, |
| ) |
| do_aemu_download = not aemu_instances |
| |
| if do_aemu_download: |
| download_aemu(api, aemu_bid, AEMU_TARGET_TO_CIPD_PLATFORM) |
| |
| summary = f"Uploaded bid:{aemu_bid} for AEMU. " if do_aemu_download else "" |
| starnix_bid = get_bid(api, True) |
| if not starnix_bid: |
| return RawResult( |
| status=common.SUCCESS, |
| summary_markdown=summary, |
| ) |
| |
| starnix_instances = api.cipd.search( |
| package_name=STARNIX_DEFAULT_PKG_TO_SEARCH, |
| tag=f"bid:{starnix_bid}", |
| test_instances=None, |
| ) |
| do_starnix_download = not starnix_instances |
| |
| if do_starnix_download: |
| download_starnix(api, starnix_bid) |
| |
| summary += f"Uploaded bid:{starnix_bid} for starnix" if do_starnix_download else "" |
| return RawResult( |
| status=common.SUCCESS, |
| summary_markdown=summary, |
| ) |
| |
| |
| def GenTests(api): |
| yield ( |
| api.test("android_downloader_do_download") |
| + api.step_data( |
| "cipd search fuchsia/third_party/android/aemu/release/linux-amd64 bid:1234567", |
| api.cipd.example_search(AEMU_DEFAULT_PKG_TO_SEARCH, []), |
| ) |
| ) |
| |
| yield ( |
| api.test("android_downloader_invalid_download", status="INFRA_FAILURE") |
| + api.step_data( |
| "cipd search fuchsia/third_party/android/aemu/release/linux-amd64 bid:1234567", |
| api.cipd.example_search(AEMU_DEFAULT_PKG_TO_SEARCH, []), |
| ) |
| + api.step_data( |
| "download AEMU artifacts and upload to CIPD.list target in emulator-linux_aarch64", |
| api.file.listdir( |
| ["sdk-repo-linux-emulator-1234567.zip", "invalid_file.json"] |
| ), |
| ) |
| ) |
| |
| yield ( |
| api.test("starnix_downloader_download") |
| + api.step_data( |
| "cipd search fuchsia_internal/starnix/android-image-amd64 bid:1234567", |
| api.cipd.example_search(STARNIX_DEFAULT_PKG_TO_SEARCH, []), |
| ) |
| + api.step_data( |
| "download Starnix artifacts and upload to CIPD.list downloaded targets", |
| api.file.listdir(["starnix_x86_64-eng", "starnix_arm64-eng"]), |
| ) |
| ) |
| |
| yield ( |
| api.test("android_downloader_no_download") |
| + api.step_data( |
| "check latest versions AEMU.get latest bid from Android build for AEMU", |
| api.cipd.example_search(AEMU_DEFAULT_PKG_TO_SEARCH, []), |
| ) |
| ) |
| yield ( |
| api.test("android_downloader_invalid_bid", status="INFRA_FAILURE") |
| + api.step_data( |
| "check latest versions AEMU.get latest bid from Android build for AEMU", |
| stdout=api.raw_io.output_text("abcd1234"), |
| ) |
| ) |
| yield ( |
| api.test("android_downloader_failed_bid", status="FAILURE") |
| + api.step_data( |
| "check latest versions AEMU.get latest bid from Android build for AEMU", |
| retcode=1, |
| stdout=api.raw_io.output_text(""), |
| ) |
| ) |
| yield ( |
| api.test("android_downloader_empty_bid", status="INFRA_FAILURE") |
| + api.step_data( |
| "check latest versions AEMU.get latest bid from Android build for AEMU", |
| stdout=api.raw_io.output_text(""), |
| ) |
| ) |
| yield ( |
| api.test("starnix_downloader_return_error") |
| + api.step_data( |
| "check latest versions starnix.get latest bid from Android build for starnix", |
| retcode=1, |
| ) |
| ) |
| yield ( |
| api.test("starnix_downloader_empty_bid") |
| + api.step_data( |
| "check latest versions starnix.get latest bid from Android build for starnix", |
| stdout=api.raw_io.output_text(""), |
| ) |
| ) |
