[gn-sdk]Publish the GN SDK to CIPD
This change publishes the GN SDK to CIPD on release alongside the
IDK (integrator development kit, formerly known as the "core" SDK)
Change-Id: I68bdca717e823299ef9b4eab55cb92967b86bc33
diff --git a/recipes/sdk.expected/release_ci.json b/recipes/sdk.expected/release_ci.json
index 67b4467..ef87bce 100644
--- a/recipes/sdk.expected/release_ci.json
+++ b/recipes/sdk.expected/release_ci.json
@@ -2288,7 +2288,7 @@
"gs://fuchsia/development/8945511751514863184/sdk/linux-amd64/core.tar.gz"
],
"infra_step": true,
- "name": "upload raw-core-sdk.upload new fuchsia-sdk 8945511751514863184",
+ "name": "upload raw-core-sdk.upload new core fuchsia-sdk 8945511751514863184",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@",
"@@@STEP_LINK@archive@https://storage.googleapis.com/fuchsia/development/8945511751514863184/sdk/linux-amd64/core.tar.gz@@@"
@@ -2452,6 +2452,66 @@
]
},
{
+ "cmd": [],
+ "name": "cipd"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "pkg-build",
+ "-pkg-def",
+ "{\"data\": [{\"dir\": \".\", \"exclude\": []}, {\"version_file\": \".versions/gn.cipd_version\"}], \"install_mode\": \"copy\", \"package\": \"fuchsia/sdk/gn/${platform}\", \"root\": \"[CLEANUP]/sdk-gn\"}",
+ "-out",
+ "[CLEANUP]/gn.cipd",
+ "-hash-algo",
+ "sha256",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "cipd.build fuchsia/sdk/gn/${platform}",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/sdk/gn/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "search",
+ "fuchsia/sdk/gn/${platform}",
+ "-tag",
+ "git_revision:c22471f4e3f842ae18dd9adec82ed9eb78ed1127",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "cipd.cipd search fuchsia/sdk/gn/${platform} git_revision:c22471f4e3f842ae18dd9adec82ed9eb78ed1127",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/sdk/gn/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "cipd.Package is up-to-date",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
"name": "$result"
}
]
\ No newline at end of file
diff --git a/recipes/sdk.expected/release_ci_new_upload.json b/recipes/sdk.expected/release_ci_new_upload.json
index 820c79a..bbf938d 100644
--- a/recipes/sdk.expected/release_ci_new_upload.json
+++ b/recipes/sdk.expected/release_ci_new_upload.json
@@ -2288,7 +2288,7 @@
"gs://fuchsia/development/8945511751514863184/sdk/linux-amd64/core.tar.gz"
],
"infra_step": true,
- "name": "upload raw-core-sdk.upload new fuchsia-sdk 8945511751514863184",
+ "name": "upload raw-core-sdk.upload new core fuchsia-sdk 8945511751514863184",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@",
"@@@STEP_LINK@archive@https://storage.googleapis.com/fuchsia/development/8945511751514863184/sdk/linux-amd64/core.tar.gz@@@"
@@ -2469,6 +2469,66 @@
]
},
{
+ "cmd": [],
+ "name": "cipd"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "pkg-build",
+ "-pkg-def",
+ "{\"data\": [{\"dir\": \".\", \"exclude\": []}, {\"version_file\": \".versions/gn.cipd_version\"}], \"install_mode\": \"copy\", \"package\": \"fuchsia/sdk/gn/${platform}\", \"root\": \"[CLEANUP]/sdk-gn\"}",
+ "-out",
+ "[CLEANUP]/gn.cipd",
+ "-hash-algo",
+ "sha256",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "cipd.build fuchsia/sdk/gn/${platform}",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/sdk/gn/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "search",
+ "fuchsia/sdk/gn/${platform}",
+ "-tag",
+ "git_revision:c22471f4e3f842ae18dd9adec82ed9eb78ed1127",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "cipd.cipd search fuchsia/sdk/gn/${platform} git_revision:c22471f4e3f842ae18dd9adec82ed9eb78ed1127",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/sdk/gn/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "cipd.Package is up-to-date",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
"name": "$result"
}
]
\ No newline at end of file
diff --git a/recipes/sdk.py b/recipes/sdk.py
index 2615a43..0b657fd 100644
--- a/recipes/sdk.py
+++ b/recipes/sdk.py
@@ -343,7 +343,7 @@
# These tests are being run for every SDK flavor.
generate_gn_path = checkout.root_dir.join('scripts', 'sdk', 'gn',
'generate.py')
- sdk_dir = api.path['cleanup'].join('sdk-gn')
+ gn_sdk_dir = api.path['cleanup'].join('sdk-gn')
test_workspace_dir = api.path['cleanup'].join('tests')
api.python(
@@ -353,7 +353,7 @@
'--archive',
full_archive_path,
'--output',
- sdk_dir,
+ gn_sdk_dir,
'--tests',
test_workspace_dir,
],
@@ -372,7 +372,17 @@
raise_on_failure=True,
)
- # Publish the core SDK.
+ # Publish the core and GN SDK.
+ # GCS publishing paths:
+ # gs://fuchsia/development/${sdk_id}/sdk/${platform}
+ # |-- core.tar.gz
+ #
+ # CIPD publishing paths (versioning is built into CIPD):
+ # https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/
+ # |-- core
+ # | `-- ${platform}
+ # `-- gn
+ # `-- ${platform}
if not publish:
return
@@ -381,12 +391,12 @@
# sdk_id.
relocate_artifacts(api, image_builds, companion_images, build.id)
+ # Publish core
gcs_archive_path = 'sdk/%s/%s-amd64' % (sdk_name, api.platform.name)
new_gcs_archive_path = 'development/%s/sdk/%s-amd64/%s.tar.gz' % (
sdk_id, api.platform.name, sdk_name)
cipd_pkg_name = '%s/sdk/%s/${platform}' % (cipd_root, sdk_name)
-
- upload_raw_sdk(
+ upload_core(
api,
sdk_name='raw-core-sdk',
gcs_archive_bucket=gcs_bucket,
@@ -400,10 +410,19 @@
build_id=str(build.id),
checkout_root=checkout.root_dir)
+ # Publish GN SDK
+ # Upload GN SDK CIPD
+ api.upload.cipd_package(
+ '%s/sdk/gn/${platform}' % cipd_root,
+ gn_sdk_dir, [api.upload.DirectoryPath(gn_sdk_dir)],
+ {'git_revision': revision},
+ repository=None,
+ extra_tags={'jiri_snapshot': sdk_id})
-def upload_raw_sdk(api, sdk_name, gcs_archive_bucket, gcs_archive_path,
- new_gcs_archive_path, cipd_pkg_name, archive_path, revision,
- update_latest, sdk_id, build_id, checkout_root):
+
+def upload_core(api, sdk_name, gcs_archive_bucket, gcs_archive_path,
+ new_gcs_archive_path, cipd_pkg_name, archive_path, revision,
+ update_latest, sdk_id, build_id, checkout_root):
sdk_dir = api.path['cleanup'].join(sdk_name)
# Extract the archive to a directory for CIPD processing.
@@ -419,7 +438,7 @@
with api.step.nest('upload ' + sdk_name):
# Upload the SDK to GCS and CIPD. Only upload sdk_id when update_latest is
# true.
- upload_archive(
+ upload_core_archive(
api,
gcs_archive_bucket=gcs_archive_bucket,
gcs_archive_path=gcs_archive_path,
@@ -435,9 +454,10 @@
)
-def upload_archive(api, gcs_archive_bucket, gcs_archive_path,
- new_gcs_archive_path, cipd_pkg_name, sdk, out_dir, revision,
- update_latest, sdk_id, build_id, checkout_root):
+def upload_core_archive(api, gcs_archive_bucket, gcs_archive_path,
+ new_gcs_archive_path, cipd_pkg_name, sdk, out_dir,
+ revision, update_latest, sdk_id, build_id,
+ checkout_root):
api.gsutil.upload(
bucket=gcs_archive_bucket,
src=sdk,
@@ -452,7 +472,7 @@
src=sdk,
dst=new_gcs_archive_path,
link_name='archive',
- name='upload new fuchsia-sdk %s' % sdk_id,
+ name='upload new core fuchsia-sdk %s' % sdk_id,
# Publicly available.
unauthenticated_url=True)
@@ -614,11 +634,12 @@
'name': '###SDK_IMAGE###',
'builder': '###SDK_IMAGE_BUILDER###',
'bucket': '###DEV_BUCKET###',
- }, {
- 'name': '###OTHER_SDK_IMAGE###',
- 'builder': '###OTHER_SDK_IMAGE_BUILDER###',
- 'bucket': '###OTHER_DEV_BUCKET###',
- }]
+ },
+ {
+ 'name': '###OTHER_SDK_IMAGE###',
+ 'builder': '###OTHER_SDK_IMAGE_BUILDER###',
+ 'bucket': '###OTHER_DEV_BUCKET###',
+ }]
yield (api.test('local_ci') + topaz_local_ci + ci_subbuilds)
yield (api.test('local_cq') + topaz_local_cq + cq_subbuilds)
@@ -628,11 +649,11 @@
api.properties(checkout_snapshot=True) + cq_subbuilds)
yield (api.test('local_ci_mac') + topaz_local_ci + api.platform.name('mac') +
ci_subbuilds)
- yield (api.test('global_ci') + topaz_global_ci + ci_subbuilds +
- api.properties(
- companion_images=companion_images,
- publish_branch=publish_branch,
- ) + failed_describe + subbuilders_properties)
+ yield (
+ api.test('global_ci') + topaz_global_ci + ci_subbuilds + api.properties(
+ companion_images=companion_images,
+ publish_branch=publish_branch,
+ ) + failed_describe + subbuilders_properties)
yield (api.test('release_ci') + topaz_release_ci + ci_subbuilds +
ci_image_builds + api.properties(
companion_images=companion_images,
@@ -644,12 +665,13 @@
ci_subbuilds_infra_failure)
yield (api.test('release_ci_image_failure') + topaz_global_ci + ci_subbuilds +
ci_image_builds_failure + api.properties(
- companion_images=companion_images, publish_branch=publish_branch) +
- describe + release_versions)
+ companion_images=companion_images,
+ publish_branch=publish_branch) + describe + release_versions)
yield (api.test('release_ci_new_upload') + topaz_release_ci + api.step_data(
'upload raw-core-sdk.cipd.cipd search fuchsia/sdk/core/${platform} ' +
- 'git_revision:%s' % revision, api.json.output({'result': []})) +
- ci_subbuilds + ci_image_builds + api.properties(
- companion_images=companion_images,
- publish_branch=publish_branch,
- ) + describe + release_versions)
+ 'git_revision:%s' % revision, api.json.output({
+ 'result': []
+ })) + ci_subbuilds + ci_image_builds + api.properties(
+ companion_images=companion_images,
+ publish_branch=publish_branch,
+ ) + describe + release_versions)
