[aemu_downloader] Add arm64 target for starnix
This change adds an arm64 target to fetch from go/ab and upload to
cipd as part of the existing aemu_downloader recipe as a short-term measure to unblock the starnix team.
Additionally, this switches to the new form of designating
artifacts and branches with the android_downloader (aemu) tool.
The aemu_downloader.py recipe will be replaced in short order by a
generic recipe that handles the currently special-cased transformations
like `unzips` and `chmod +x` as configuration parameters.
Bug: 116835
Change-Id: I032e614b5c1cb0ec02398e7ec7cb19b73374bff2
Reviewed-on: https://fuchsia-review.googlesource.com/c/infra/recipes/+/835077
Reviewed-by: Ina Huh <ihuh@google.com>
Fuchsia-Auto-Submit: Catherine Duncan <catduncan@google.com>
Commit-Queue: Auto-Submit <auto-submit@fuchsia-infra.iam.gserviceaccount.com>
diff --git a/recipe_modules/android_downloader/api.py b/recipe_modules/android_downloader/api.py
index bcf166d..673085e 100644
--- a/recipe_modules/android_downloader/api.py
+++ b/recipe_modules/android_downloader/api.py
@@ -28,7 +28,7 @@
class AEMUDownloaderAPI(recipe_api.RecipeApi):
"""APIs for running android_downloader."""
- def get_latest_build_id(self, starnix, step_name, **kwargs):
+ def get_latest_build_id(self, step_name, artifacts, branch, **kwargs):
"""Runs android_downloader tool to get the latest Android build ID
where all targets we use has been built successfully.
@@ -41,6 +41,8 @@
Args:
step_name (str): Name of the step.
+ artifacts (dict<str,array<str>>): Artifacts to check the existence of.
+ branch (str): Which Android branch to check for builds.
kwargs (dict): Passed to self.m.step.
"""
args = [
@@ -53,21 +55,28 @@
"--",
self._android_downloader_tool,
"-find_bid",
+ "-branch",
+ branch,
]
- if starnix:
- args += ["-starnix"]
- else:
- args += ["-aemu"]
+
+ for target, paths in artifacts.items():
+ for path in paths:
+ args.append("-artifact")
+ args.append(f"{target},{path}")
step = self._run(step_name, args, **kwargs)
return step
- def download(self, starnix, step_name, android_build_id, output_root, **kwargs):
+ def download(
+ self, step_name, android_build_id, artifacts, branch, output_root, **kwargs
+ ):
"""Runs android_downloader to download artifacts from a specified Android build ID.
Args:
step_name (str): Name of the step.
android_build_id (str): Android build id to use for artifact download.
+ artifacts (dict<str,array<str>>): Artifacts to download.
+ branch (str): Which Android branch to download artifacts from.
output_root (Path): Path root to store downloaded artifacts.
kwargs (dict): Passed to self.m.step.
"""
@@ -84,11 +93,14 @@
android_build_id,
"-output",
output_root,
+ "-branch",
+ branch,
]
- if starnix:
- args += ["-starnix"]
- else:
- args += ["-aemu"]
+
+ for target, paths in artifacts.items():
+ for path in paths:
+ args.append("-artifact")
+ args.append(f"{target},{path}")
step = self._run(step_name, args, **kwargs)
return step
diff --git a/recipe_modules/android_downloader/tests/full.expected/basic.json b/recipe_modules/android_downloader/tests/full.expected/basic.json
index fd22d62..c006412 100644
--- a/recipe_modules/android_downloader/tests/full.expected/basic.json
+++ b/recipe_modules/android_downloader/tests/full.expected/basic.json
@@ -172,7 +172,10 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "branch-name",
+ "-artifact",
+ "target,path/to/file"
],
"name": "find bid"
},
@@ -190,44 +193,14 @@
"12345",
"-output",
"[TMP_BASE]",
- "-aemu"
+ "-branch",
+ "branch-name",
+ "-artifact",
+ "target,path/to/file"
],
"name": "download"
},
{
- "cmd": [
- "[START_DIR]/cipd_tool/path/to/luci-auth/version%3Apinned-version/luci-auth",
- "context",
- "-scopes",
- "https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/firebase https://www.googleapis.com/auth/gerritcodereview https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/androidbuild.internal",
- "-act-as-service-account",
- "aemu-artifact-mover@fuchsia-cloud-api-for-test.iam.gserviceaccount.com",
- "--",
- "[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
- "-find_bid",
- "-starnix"
- ],
- "name": "find bid (2)"
- },
- {
- "cmd": [
- "[START_DIR]/cipd_tool/path/to/luci-auth/version%3Apinned-version/luci-auth",
- "context",
- "-scopes",
- "https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/firebase https://www.googleapis.com/auth/gerritcodereview https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/androidbuild.internal",
- "-act-as-service-account",
- "aemu-artifact-mover@fuchsia-cloud-api-for-test.iam.gserviceaccount.com",
- "--",
- "[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
- "-bid",
- "12345",
- "-output",
- "[TMP_BASE]",
- "-starnix"
- ],
- "name": "download (2)"
- },
- {
"name": "$result"
}
]
\ No newline at end of file
diff --git a/recipe_modules/android_downloader/tests/full.py b/recipe_modules/android_downloader/tests/full.py
index a62f759..e8ca110 100644
--- a/recipe_modules/android_downloader/tests/full.py
+++ b/recipe_modules/android_downloader/tests/full.py
@@ -10,24 +10,16 @@
def RunSteps(api):
api.android_downloader.get_latest_build_id(
- False,
- "find bid",
+ step_name="find bid",
+ branch="branch-name",
+ artifacts={"target": ["path/to/file"]},
)
api.android_downloader.download(
- False,
- "download",
- "12345",
- api.path["tmp_base"],
- )
- api.android_downloader.get_latest_build_id(
- True,
- "find bid",
- )
- api.android_downloader.download(
- True,
- "download",
- "12345",
- api.path["tmp_base"],
+ step_name="download",
+ android_build_id="12345",
+ artifacts={"target": ["path/to/file"]},
+ branch="branch-name",
+ output_root=api.path["tmp_base"],
)
diff --git a/recipes/contrib/aemu_downloader.expected/android_downloader_do_download.json b/recipes/contrib/aemu_downloader.expected/android_downloader_do_download.json
index 8fb4002..baf84bc 100644
--- a/recipes/contrib/aemu_downloader.expected/android_downloader_do_download.json
+++ b/recipes/contrib/aemu_downloader.expected/android_downloader_do_download.json
@@ -182,7 +182,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
@@ -243,7 +252,16 @@
"1234567",
"-output",
"[CLEANUP]/1234567",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "download AEMU artifacts and upload to CIPD.download bid: 1234567",
"~followup_annotations": [
@@ -562,7 +580,40 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-starnix"
+ "-branch",
+ "git_starnix-dev",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_x86_64-img-{bid}.zip",
+ "-artifact",
+ "starnix_x86_64-eng,bin/adb",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_x86_64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_x86_64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_x86_64-eng,build-id.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_arm64-img-{bid}.zip",
+ "-artifact",
+ "starnix_arm64-eng,bin/adb",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_arm64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_arm64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_arm64-eng,build-id.zip"
],
"name": "check latest versions starnix.get latest bid from Android build for starnix",
"~followup_annotations": [
diff --git a/recipes/contrib/aemu_downloader.expected/android_downloader_empty_bid.json b/recipes/contrib/aemu_downloader.expected/android_downloader_empty_bid.json
index e7f9001..98fcc80 100644
--- a/recipes/contrib/aemu_downloader.expected/android_downloader_empty_bid.json
+++ b/recipes/contrib/aemu_downloader.expected/android_downloader_empty_bid.json
@@ -185,7 +185,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
diff --git a/recipes/contrib/aemu_downloader.expected/android_downloader_failed_bid.json b/recipes/contrib/aemu_downloader.expected/android_downloader_failed_bid.json
index 6d2ffc2..6d2dce0 100644
--- a/recipes/contrib/aemu_downloader.expected/android_downloader_failed_bid.json
+++ b/recipes/contrib/aemu_downloader.expected/android_downloader_failed_bid.json
@@ -185,7 +185,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
diff --git a/recipes/contrib/aemu_downloader.expected/android_downloader_invalid_bid.json b/recipes/contrib/aemu_downloader.expected/android_downloader_invalid_bid.json
index 10c3d26..1d71a44 100644
--- a/recipes/contrib/aemu_downloader.expected/android_downloader_invalid_bid.json
+++ b/recipes/contrib/aemu_downloader.expected/android_downloader_invalid_bid.json
@@ -185,7 +185,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
diff --git a/recipes/contrib/aemu_downloader.expected/android_downloader_invalid_download.json b/recipes/contrib/aemu_downloader.expected/android_downloader_invalid_download.json
index 0c71cb7..f82bed8 100644
--- a/recipes/contrib/aemu_downloader.expected/android_downloader_invalid_download.json
+++ b/recipes/contrib/aemu_downloader.expected/android_downloader_invalid_download.json
@@ -182,7 +182,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
@@ -246,7 +255,16 @@
"1234567",
"-output",
"[CLEANUP]/1234567",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "download AEMU artifacts and upload to CIPD.download bid: 1234567",
"~followup_annotations": [
diff --git a/recipes/contrib/aemu_downloader.expected/android_downloader_no_download.json b/recipes/contrib/aemu_downloader.expected/android_downloader_no_download.json
index 5c24d3a..ee414d5 100644
--- a/recipes/contrib/aemu_downloader.expected/android_downloader_no_download.json
+++ b/recipes/contrib/aemu_downloader.expected/android_downloader_no_download.json
@@ -182,7 +182,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
@@ -227,7 +236,40 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-starnix"
+ "-branch",
+ "git_starnix-dev",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_x86_64-img-{bid}.zip",
+ "-artifact",
+ "starnix_x86_64-eng,bin/adb",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_x86_64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_x86_64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_x86_64-eng,build-id.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_arm64-img-{bid}.zip",
+ "-artifact",
+ "starnix_arm64-eng,bin/adb",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_arm64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_arm64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_arm64-eng,build-id.zip"
],
"name": "check latest versions starnix.get latest bid from Android build for starnix",
"~followup_annotations": [
diff --git a/recipes/contrib/aemu_downloader.expected/starnix_downloader_download.json b/recipes/contrib/aemu_downloader.expected/starnix_downloader_download.json
index b0090df..a2613ec 100644
--- a/recipes/contrib/aemu_downloader.expected/starnix_downloader_download.json
+++ b/recipes/contrib/aemu_downloader.expected/starnix_downloader_download.json
@@ -182,7 +182,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
@@ -227,7 +236,40 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-starnix"
+ "-branch",
+ "git_starnix-dev",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_x86_64-img-{bid}.zip",
+ "-artifact",
+ "starnix_x86_64-eng,bin/adb",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_x86_64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_x86_64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_x86_64-eng,build-id.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_arm64-img-{bid}.zip",
+ "-artifact",
+ "starnix_arm64-eng,bin/adb",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_arm64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_arm64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_arm64-eng,build-id.zip"
],
"name": "check latest versions starnix.get latest bid from Android build for starnix",
"~followup_annotations": [
@@ -288,7 +330,40 @@
"1234567",
"-output",
"[CLEANUP]/1234567",
- "-starnix"
+ "-branch",
+ "git_starnix-dev",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_x86_64-img-{bid}.zip",
+ "-artifact",
+ "starnix_x86_64-eng,bin/adb",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_x86_64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_x86_64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_x86_64-eng,build-id.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_arm64-img-{bid}.zip",
+ "-artifact",
+ "starnix_arm64-eng,bin/adb",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_arm64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_arm64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_arm64-eng,build-id.zip"
],
"name": "download Starnix artifacts and upload to CIPD.download starnix bid: 1234567",
"~followup_annotations": [
@@ -309,6 +384,7 @@
"name": "download Starnix artifacts and upload to CIPD.list downloaded targets",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@listdir@[CLEANUP]/1234567/starnix_arm64-eng@@@",
"@@@STEP_LOG_LINE@listdir@[CLEANUP]/1234567/starnix_x86_64-eng@@@",
"@@@STEP_LOG_END@listdir@@@"
]
@@ -337,6 +413,187 @@
"-u",
"RECIPE_MODULE[recipe_engine::archive]/resources/extract.py",
"--json-input",
+ "{\"archive_file\": \"[CLEANUP]/1234567/starnix_arm64-eng/starnix_arm64-img-1234567.zip\", \"include_files\": [], \"output\": \"[CLEANUP]/1234567/starnix_arm64-eng\", \"safe_mode\": true}",
+ "--json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "download Starnix artifacts and upload to CIPD.unzip [CLEANUP]/1234567/starnix_arm64-eng/starnix_arm64-img-1234567.zip to [CLEANUP]/1234567/starnix_arm64-eng",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_TEXT@<br/>extracted 1337 files - 50159.75 MB@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"extracted\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bytes\": 50159747054, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"filecount\": 1337@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "remove",
+ "[CLEANUP]/1234567/starnix_arm64-eng/starnix_arm64-img-1234567.zip"
+ ],
+ "infra_step": true,
+ "name": "download Starnix artifacts and upload to CIPD.remove image zip file",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::archive]/resources/extract.py",
+ "--json-input",
+ "{\"archive_file\": \"[CLEANUP]/1234567/starnix_arm64-eng/build-id.zip\", \"include_files\": [], \"output\": \"[CLEANUP]/1234567-build-id\", \"safe_mode\": true}",
+ "--json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "download Starnix artifacts and upload to CIPD.unzip [CLEANUP]/1234567/starnix_arm64-eng/build-id.zip to [CLEANUP]/1234567-build-id",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_TEXT@<br/>extracted 1337 files - 50159.75 MB@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"extracted\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bytes\": 50159747054, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"filecount\": 1337@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "remove",
+ "[CLEANUP]/1234567/starnix_arm64-eng/build-id.zip"
+ ],
+ "infra_step": true,
+ "name": "download Starnix artifacts and upload to CIPD.remove symbol zip file",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "listdir",
+ "[CLEANUP]/1234567/starnix_arm64-eng/bin"
+ ],
+ "infra_step": true,
+ "name": "download Starnix artifacts and upload to CIPD.list binaries",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@listdir@[CLEANUP]/1234567/starnix_arm64-eng/bin/adb@@@",
+ "@@@STEP_LOG_END@listdir@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "chmod",
+ "+x",
+ "[CLEANUP]/1234567/starnix_arm64-eng/bin/adb"
+ ],
+ "name": "download Starnix artifacts and upload to CIPD.make adb executable",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "create",
+ "-pkg-def",
+ "{\"data\": [{\"dir\": \".\", \"exclude\": []}], \"install_mode\": \"copy\", \"package\": \"fuchsia_internal/starnix/android-image-arm64\", \"root\": \"[CLEANUP]/1234567/starnix_arm64-eng\"}",
+ "-hash-algo",
+ "sha256",
+ "-ref",
+ "latest",
+ "-tag",
+ "bid:1234567",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "download Starnix artifacts and upload to CIPD.create fuchsia_internal/starnix/android-image-arm64",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia_internal/starnix/android-image-arm64\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LINK@40-chars-fake-of-the-package-instance_id@https://chrome-infra-packages.appspot.com/p/fuchsia_internal/starnix/android-image-arm64/+/40-chars-fake-of-the-package-instance_id@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "create",
+ "-pkg-def",
+ "{\"data\": [{\"dir\": \".\", \"exclude\": []}], \"install_mode\": \"copy\", \"package\": \"fuchsia_internal/starnix/android-image-debug-symbols-arm64\", \"root\": \"[CLEANUP]/1234567-build-id\"}",
+ "-hash-algo",
+ "sha256",
+ "-ref",
+ "latest",
+ "-tag",
+ "bid:1234567",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "download Starnix artifacts and upload to CIPD.create fuchsia_internal/starnix/android-image-debug-symbols-arm64",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia_internal/starnix/android-image-debug-symbols-arm64\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LINK@40-chars-fake-of-the-package-instance_id@https://chrome-infra-packages.appspot.com/p/fuchsia_internal/starnix/android-image-debug-symbols-arm64/+/40-chars-fake-of-the-package-instance_id@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CLEANUP]/1234567-build-id"
+ ],
+ "infra_step": true,
+ "name": "download Starnix artifacts and upload to CIPD.ensure symbol dir (2)",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::archive]/resources/extract.py",
+ "--json-input",
"{\"archive_file\": \"[CLEANUP]/1234567/starnix_x86_64-eng/starnix_x86_64-img-1234567.zip\", \"include_files\": [], \"output\": \"[CLEANUP]/1234567/starnix_x86_64-eng\", \"safe_mode\": true}",
"--json-output",
"/path/to/tmp/json"
@@ -365,7 +622,7 @@
"[CLEANUP]/1234567/starnix_x86_64-eng/starnix_x86_64-img-1234567.zip"
],
"infra_step": true,
- "name": "download Starnix artifacts and upload to CIPD.remove image zip file",
+ "name": "download Starnix artifacts and upload to CIPD.remove image zip file (2)",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -404,7 +661,7 @@
"[CLEANUP]/1234567/starnix_x86_64-eng/build-id.zip"
],
"infra_step": true,
- "name": "download Starnix artifacts and upload to CIPD.remove symbol zip file",
+ "name": "download Starnix artifacts and upload to CIPD.remove symbol zip file (2)",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -420,7 +677,7 @@
"[CLEANUP]/1234567/starnix_x86_64-eng/bin"
],
"infra_step": true,
- "name": "download Starnix artifacts and upload to CIPD.list binaries",
+ "name": "download Starnix artifacts and upload to CIPD.list binaries (2)",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@",
"@@@STEP_LOG_LINE@listdir@[CLEANUP]/1234567/starnix_x86_64-eng/bin/adb@@@",
@@ -433,7 +690,7 @@
"+x",
"[CLEANUP]/1234567/starnix_x86_64-eng/bin/adb"
],
- "name": "download Starnix artifacts and upload to CIPD.make adb executable",
+ "name": "download Starnix artifacts and upload to CIPD.make adb executable (2)",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
diff --git a/recipes/contrib/aemu_downloader.expected/starnix_downloader_empty_bid.json b/recipes/contrib/aemu_downloader.expected/starnix_downloader_empty_bid.json
index e3c7a11..7cff745 100644
--- a/recipes/contrib/aemu_downloader.expected/starnix_downloader_empty_bid.json
+++ b/recipes/contrib/aemu_downloader.expected/starnix_downloader_empty_bid.json
@@ -182,7 +182,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
@@ -227,7 +236,40 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-starnix"
+ "-branch",
+ "git_starnix-dev",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_x86_64-img-{bid}.zip",
+ "-artifact",
+ "starnix_x86_64-eng,bin/adb",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_x86_64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_x86_64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_x86_64-eng,build-id.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_arm64-img-{bid}.zip",
+ "-artifact",
+ "starnix_arm64-eng,bin/adb",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_arm64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_arm64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_arm64-eng,build-id.zip"
],
"name": "check latest versions starnix.get latest bid from Android build for starnix",
"~followup_annotations": [
diff --git a/recipes/contrib/aemu_downloader.expected/starnix_downloader_return_error.json b/recipes/contrib/aemu_downloader.expected/starnix_downloader_return_error.json
index 5c24d3a..ee414d5 100644
--- a/recipes/contrib/aemu_downloader.expected/starnix_downloader_return_error.json
+++ b/recipes/contrib/aemu_downloader.expected/starnix_downloader_return_error.json
@@ -182,7 +182,16 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-aemu"
+ "-branch",
+ "aosp-emu-master-dev",
+ "-artifact",
+ "emulator-linux_x64_internal,sdk-repo-linux-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-linux_aarch64,sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_x64,sdk-repo-darwin-emulator-{bid}.zip",
+ "-artifact",
+ "emulator-mac_aarch64,sdk-repo-darwin_aarch64-emulator-{bid}.zip"
],
"name": "check latest versions AEMU.get latest bid from Android build for AEMU",
"~followup_annotations": [
@@ -227,7 +236,40 @@
"--",
"[START_DIR]/cipd_tool/path/to/aemu/version%3Apinned-version/aemu",
"-find_bid",
- "-starnix"
+ "-branch",
+ "git_starnix-dev",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_x86_64-img-{bid}.zip",
+ "-artifact",
+ "starnix_x86_64-eng,bin/adb",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl",
+ "-artifact",
+ "starnix_x86_64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_x86_64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_x86_64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_x86_64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_x86_64-eng,build-id.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_arm64-img-{bid}.zip",
+ "-artifact",
+ "starnix_arm64-eng,bin/adb",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl",
+ "-artifact",
+ "starnix_arm64-eng,bin/aidl_rust_glue.py",
+ "-artifact",
+ "starnix_arm64-eng,lib64/libc++.so",
+ "-artifact",
+ "starnix_arm64-eng,art-run-tst-target-data-merged.zip",
+ "-artifact",
+ "starnix_arm64-eng,starnix_testcases.img",
+ "-artifact",
+ "starnix_arm64-eng,build-id.zip"
],
"name": "check latest versions starnix.get latest bid from Android build for starnix",
"~followup_annotations": [
diff --git a/recipes/contrib/aemu_downloader.py b/recipes/contrib/aemu_downloader.py
index 7e16d09..7b61e4f 100644
--- a/recipes/contrib/aemu_downloader.py
+++ b/recipes/contrib/aemu_downloader.py
@@ -3,7 +3,8 @@
# found in the LICENSE file.
"""Downloads AEMU artifacts from go/ab then uploads to CIPD."""
-# TODO(fxbug.dev/116835): This recipe should be deleted in favor of using 3pp.
+# TODO(fxbug.dev/116835): Write new recipe that makes this workflow generic,
+# migrate existing package
from PB.recipe_engine.result import RawResult
from PB.go.chromium.org.luci.buildbucket.proto import common
@@ -22,6 +23,8 @@
STARNIX_CIPD_PKG_ROOT = "fuchsia_internal/starnix"
AEMU_DEFAULT_PKG_TO_SEARCH = f"{AEMU_CIPD_PKG_ROOT}/release/linux-amd64"
STARNIX_DEFAULT_PKG_TO_SEARCH = f"{STARNIX_CIPD_PKG_ROOT}/android-image-amd64"
+AEMU_ANDROID_BRANCH = "aosp-emu-master-dev"
+STARNIX_ANDROID_BRANCH = "git_starnix-dev"
# The downloaded artifacts follow the file directory pattern
# [download_root]/target/artifact. An example would be
@@ -34,9 +37,49 @@
"emulator-mac_aarch64": "mac-arm64",
}
+AEMU_ARTIFACTS = {
+ "emulator-linux_x64_internal": [
+ "sdk-repo-linux-emulator-{bid}.zip",
+ ],
+ "emulator-linux_aarch64": [
+ "sdk-repo-linux_aarch64-emulator-{bid}.zip",
+ ],
+ "emulator-mac_x64": [
+ "sdk-repo-darwin-emulator-{bid}.zip",
+ ],
+ "emulator-mac_aarch64": [
+ "sdk-repo-darwin_aarch64-emulator-{bid}.zip",
+ ],
+}
+
STARNIX_TARGET_TO_CIPD_PLATFORM = {
"starnix_x86_64-eng": "android-image-amd64",
"starnix_x86_64-eng-symbols": "android-image-debug-symbols-amd64",
+ "starnix_arm64-eng": "android-image-arm64",
+ "starnix_arm64-eng-symbols": "android-image-debug-symbols-arm64",
+}
+
+STARNIX_ARTIFACTS = {
+ "starnix_x86_64-eng": [
+ "starnix_x86_64-img-{bid}.zip",
+ "bin/adb",
+ "bin/aidl",
+ "bin/aidl_rust_glue.py",
+ "lib64/libc++.so",
+ "art-run-tst-target-data-merged.zip",
+ "starnix_testcases.img",
+ "build-id.zip",
+ ],
+ "starnix_arm64-eng": [
+ "starnix_arm64-img-{bid}.zip",
+ "bin/adb",
+ "bin/aidl",
+ "bin/aidl_rust_glue.py",
+ "lib64/libc++.so",
+ "art-run-tst-target-data-merged.zip",
+ "starnix_testcases.img",
+ "build-id.zip",
+ ],
}
@@ -44,8 +87,9 @@
"""Gets the latest version of the target"""
with api.step.nest(f"check latest versions {'starnix' if starnix else 'AEMU'}"):
step = api.android_downloader.get_latest_build_id(
- starnix,
step_name=f"get latest bid from Android build for {'starnix' if starnix else 'AEMU'}",
+ artifacts=STARNIX_ARTIFACTS if starnix else AEMU_ARTIFACTS,
+ branch=STARNIX_ANDROID_BRANCH if starnix else AEMU_ANDROID_BRANCH,
stdout=api.raw_io.output_text(),
step_test_data=lambda: api.raw_io.test_api.stream_output_text("1234567"),
ok_ret="any" if starnix else (0,),
@@ -72,9 +116,10 @@
download_root = api.path["cleanup"].join(bid)
api.file.ensure_directory("ensure download dir", download_root)
api.android_downloader.download(
- starnix=False,
step_name=f"download bid: {bid}",
android_build_id=bid,
+ artifacts=AEMU_ARTIFACTS,
+ branch=AEMU_ANDROID_BRANCH,
output_root=download_root,
)
@@ -137,8 +182,9 @@
download_root = api.path["cleanup"].join(bid)
api.file.ensure_directory("ensure download dir", download_root)
api.android_downloader.download(
- starnix=True,
step_name=f"download starnix bid: {bid}",
+ artifacts=STARNIX_ARTIFACTS,
+ branch=STARNIX_ANDROID_BRANCH,
android_build_id=bid,
output_root=download_root,
)
@@ -151,7 +197,10 @@
target_name = api.path.basename(target)
cipd_platform = STARNIX_TARGET_TO_CIPD_PLATFORM[target_name]
cipd_pkg_dest = f"{STARNIX_CIPD_PKG_ROOT}/{cipd_platform}"
+
image_name = f"starnix_x86_64-img-{bid}.zip"
+ if "arm64" in target_name:
+ image_name = f"starnix_arm64-img-{bid}.zip"
image_zip = f"{target}/{image_name}"
cipd_symbols_platform = STARNIX_TARGET_TO_CIPD_PLATFORM[
@@ -307,7 +356,7 @@
)
+ api.step_data(
"download Starnix artifacts and upload to CIPD.list downloaded targets",
- api.file.listdir(["starnix_x86_64-eng"]),
+ api.file.listdir(["starnix_x86_64-eng", "starnix_arm64-eng"]),
)
)
