| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling CIPD prebuilts into Fuchsia.""" |
| |
| import os |
| import re |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/build', |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/jiri', |
| 'fuchsia/upload_debug_symbols', |
| 'recipe_engine/archive', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = { |
| 'project': |
| Property(kind=str, help='Jiri remote manifest project', default=None), |
| 'manifest': |
| Property(kind=str, help='Jiri manifest to use'), |
| 'remote': |
| Property(kind=str, help='Remote manifest repository'), |
| 'import_in': |
| Property( |
| kind=str, help='Path to the manifest to edit relative to $project'), |
| 'packages': |
| Property( |
| kind=List(str), |
| help='The list of CIPD packages to update in $import_in'), |
| 'debug_symbol_attribute': |
| Property( |
| kind=str, |
| default='debug-symbols', |
| help='Jiri attribute to match debug symbol packages'), |
| 'debug_symbol_gcs_buckets': |
| Property( |
| kind=List(str), |
| default=(), |
| help='GCS buckets to upload debug symbols upon successful roll'), |
| 'lockfiles': |
| Property( |
| kind=List(str), |
| default=(), |
| help='The list of lockfiles to update in "${manifest}=${lockfile}" format' |
| ), |
| 'dry_run': |
| Property( |
| kind=bool, |
| default=False, |
| help='Whether to dry-run the auto-roller (CQ+1 and abandon the change)' |
| ), |
| 'tag': |
| Property( |
| kind=str, |
| default='version', |
| help='A CIPD tag common to all $packages where a common version can be extracted' |
| ), |
| 'ref': |
| Property( |
| kind=str, |
| default='latest', |
| help='A common CIPD ref to resolve when rolling a set of packages'), |
| 'owners': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('The owners responsible for watching this roller ' |
| '(example: "username@google.com").')), |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {roller} CIPD packages: |
| |
| {packages} |
| |
| From: {old_version} |
| To: {version} |
| |
| Cq-Cl-Tag: roller-builder:{builder} |
| Cq-Cl-Tag: roller-bid:{build_id} |
| CQ-Do-Not-Cancel-Tryjobs: true""" |
| |
| CIPD_URL = 'https://chrome-infra-packages.appspot.com/p/{package}/+/{version}' |
| |
| |
| def get_tags_from_cipd(api, cipd_pkg, tag, ref): |
| cipd_tags = api.cipd.describe(cipd_pkg, ref).tags |
| return set( |
| [cipd_tag.tag for cipd_tag in cipd_tags if cipd_tag.tag.startswith(tag)]) |
| |
| |
| def fetch_and_upload_debug_symbols(api, project, import_in, remote, project_dir, |
| packages, debug_symbol_attribute, |
| debug_symbol_gcs_buckets): |
| """ |
| Fetch debug symbol archives, unpack them, and upload debug symbols. |
| |
| Args: |
| project (str): Jiri remote manifest project. |
| import_in (str): Path to the edited manifest relative to $project |
| containing debug symbol packages. |
| remote (str): Remote manifest repository. |
| project_dir (Path): Project root path of $import_in. |
| packages (seq(str)): The list of CIPD packages updated in $import_in. |
| debug_symbol_attribute (str): Jiri attribute to match debug symbol packages. |
| debug_symbol_gcs_buckets (seq(str)): GCS buckets to upload debug symbols to. |
| """ |
| with api.context(infra_steps=True): |
| api.jiri.init( |
| use_lock_file=True, |
| attributes=(debug_symbol_attribute,), |
| ) |
| # Fetch debug symbol packages using locally edited manifest. |
| api.jiri.import_manifest( |
| manifest=import_in, |
| remote=remote, |
| name=project, |
| ) |
| api.jiri.fetch_packages(local_manifest=True) |
| |
| with api.step.nest('build'): |
| gn_results = api.build.gen( |
| checkout_root=api.path['start_dir'], |
| fuchsia_build_dir=api.path['start_dir'].join('out', 'default'), |
| target='x64', |
| build_type='debug', |
| product='products/bringup.gni', |
| # //bundles:infratools is necessary to build upload_debug_symbols. |
| packages=['//bundles:infratools'], |
| ) |
| |
| upload_debug_symbols_target = os.path.relpath( |
| str(gn_results.tool('upload_debug_symbols')), |
| str(gn_results.fuchsia_build_dir), |
| ) |
| api.build.ninja( |
| checkout_root=api.path['start_dir'], |
| gn_results=gn_results, |
| build_zircon=False, |
| targets=[upload_debug_symbols_target], |
| ) |
| |
| build_id_dirs = [] |
| for package in packages: |
| # Find archives for each debug symbol package. |
| with api.context(cwd=project_dir): |
| package_def = api.jiri.read_manifest_element( |
| manifest=import_in, |
| element_type='package', |
| element_name=package, |
| ) |
| # Skip non debug symbol packages. |
| if debug_symbol_attribute not in package_def.get('attributes', ''): |
| continue |
| |
| package_path = api.path['start_dir'].join(package_def['path']) |
| archives = api.file.glob_paths( |
| name='find archives for %s' % package, |
| source=package_path, |
| pattern='**/*.tar.bz2', |
| test_data=(package_path.join('symbols.tar.bz2'),), |
| ) |
| |
| # Unpack archives into .build-id dirs. |
| for archive in archives: |
| # Extract API requires a unique, non-existent directory. |
| archive_basename = os.path.basename(api.path.abspath(archive)) |
| output_dir = api.path['start_dir'].join(package, archive_basename) |
| api.archive.extract( |
| step_name='extract %s' % archive, |
| archive_file=archive, |
| output=output_dir, |
| ) |
| build_id_dirs.append(output_dir) |
| |
| for debug_symbol_gcs_bucket in debug_symbol_gcs_buckets: |
| api.upload_debug_symbols( |
| step_name='upload debug symbols', |
| upload_debug_symbols_path=gn_results.tool('upload_debug_symbols'), |
| bucket=debug_symbol_gcs_bucket, |
| build_id_dirs=build_id_dirs, |
| ) |
| |
| |
| def _get_platform_specific_packages(package, output): |
| platform_regex = '(?<=' + package.replace('${platform}', |
| r'\${platform=).*(?=})') |
| pattern = re.compile(platform_regex) |
| match = pattern.search(output) |
| if match: |
| platforms = match.group(0).split(',') |
| return [package.replace('${platform}', platform) for platform in platforms] |
| return [package] |
| |
| |
| def _append_urls(packages, old_version, new_version): |
| package_line = '{package} old:{old} new:{new}' |
| packages_with_urls = [] |
| for package in packages: |
| if '${platform}' in package: |
| packages_with_urls.append(package) |
| else: |
| packages_with_urls.append( |
| package_line.format( |
| old=CIPD_URL.format(package=package, version=old_version), |
| new=CIPD_URL.format(package=package, version=new_version), |
| package=package)) |
| return packages_with_urls |
| |
| |
| def RunSteps(api, project, manifest, remote, import_in, packages, |
| debug_symbol_attribute, debug_symbol_gcs_buckets, lockfiles, |
| dry_run, tag, ref, owners): |
| with api.context(infra_steps=True): |
| if owners: |
| owners_step = api.step('owners', None) |
| owners_step.presentation.step_summary_text = ', '.join(owners) |
| |
| api.jiri.init(use_lock_file=True) |
| api.jiri.import_manifest(manifest, remote, project) |
| api.jiri.update(run_hooks=False) |
| api.jiri.run_hooks() |
| |
| project_dir = api.path['start_dir'].join(*project.split('/')) |
| |
| # Populate the set of tags. |
| tags = get_tags_from_cipd(api, packages[0], tag, ref) |
| |
| # Get the intersection of tags. |
| if (len(packages) > 1): |
| for cipd_pkg in packages[1:]: |
| tags = tags.intersection(get_tags_from_cipd(api, cipd_pkg, tag, ref)) |
| |
| # Check if the intersection of tag exists. |
| if not tags: |
| api.step('unable to find common <tag> to roll', None) |
| return |
| |
| # All the instances of tag point to the same 'instance_id' |
| # Take the first one. |
| version = list(tags)[0] |
| |
| with api.context(cwd=project_dir): |
| changes = api.jiri.edit_manifest( |
| import_in, packages=[(package, version) for package in packages]) |
| |
| if not changes['packages']: |
| api.step('manifest up-to-date; nothing to roll', None) |
| return |
| |
| old_version = changes['packages'][0]['old_version'] |
| exact_packages = set() |
| |
| # Test data for jiri.resolve |
| package_test_data = [ |
| ('@Subdir prebuilt/tools/buildbucket\n' + |
| package[:package.index('${platform}')] + |
| '${platform=linux-amd64,mac-amd64} git_revision:aa2dae..') |
| for package in packages |
| if '${platform}' in package |
| ] |
| test_data = '\n'.join(package_test_data) |
| |
| # Update the lockfiles. |
| for lock_entry in lockfiles: |
| fields = lock_entry.split('=') |
| manifest = fields[0] |
| lock = fields[1] |
| resolve_output = api.jiri.resolve( |
| local_manifest=True, |
| output=lock, |
| manifests=[manifest], |
| step_test_data=test_data).stdout |
| for p in packages: |
| if '${platform}' in p: |
| platform_pkgs = _get_platform_specific_packages(p, resolve_output) |
| exact_packages = exact_packages.union(platform_pkgs) |
| else: |
| exact_packages.add(p) |
| |
| exact_packages = sorted(exact_packages) |
| packages_with_urls = _append_urls(exact_packages, old_version, version) |
| |
| roller_name = api.buildbucket.builder_name.rstrip('-roller') |
| message = COMMIT_MESSAGE.format( |
| roller=roller_name, |
| packages='\n'.join(packages_with_urls), |
| old_version=old_version, |
| version=version, |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| |
| # Land the changes. |
| rolled = api.auto_roller.attempt_roll( |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| ) |
| |
| # If roll succeeded, upload any debug symbols that were rolled. |
| # TODO(fxb/37432): Upload debug symbols with artifactory. |
| if rolled and debug_symbol_gcs_buckets: |
| with api.step.nest('fetch and upload debug symbols'): |
| fetch_and_upload_debug_symbols( |
| api=api, |
| project=project, |
| import_in=import_in, |
| remote=remote, |
| project_dir=project_dir, |
| packages=packages, |
| debug_symbol_attribute=debug_symbol_attribute, |
| debug_symbol_gcs_buckets=debug_symbol_gcs_buckets, |
| ) |
| |
| |
| def GenTests(api): |
| default_packages = [ |
| 'chromium/fuchsia/webrunner-arm64', 'chromium/fuchsia/webrunner-amd64', |
| 'chromium/fuchsia/fidl' |
| ] |
| debug_symbol_packages = [ |
| 'chromium/fuchsia/webrunner-debug-symbols-arm64', |
| 'chromium/fuchsia/webrunner-debug-symbols-amd64' |
| ] |
| platform_packages = [ |
| 'fuchsia/tools/bb/${platform}', 'fuchsia/tools/luci/gsutil/${platform}' |
| ] |
| default_lockfiles = [ |
| 'integration/flower=integration/jiri.lock', |
| ] |
| |
| default_properties = api.properties( |
| project='integration', |
| manifest='minimal', |
| remote='https://fuchsia.googlesource.com', |
| import_in='chromium/chromium', |
| packages=default_packages, |
| lockfiles=default_lockfiles, |
| owners=['nobody@google.com', 'noreply@google.com']) |
| |
| debug_symbols_properties = api.properties( |
| project='integration', |
| manifest='minimal', |
| remote='https://fuchsia.googlesource.com', |
| import_in='chromium/chromium', |
| packages=default_packages + debug_symbol_packages, |
| debug_symbol_gcs_buckets=('foo-bucket', 'bar-bucket'), |
| lockfiles=default_lockfiles, |
| ) |
| |
| platform_pkg_properties = api.properties( |
| project='integration', |
| manifest='minimal', |
| remote='https://fuchsia.googlesource.com', |
| import_in='fuchsia/prebuilts', |
| packages=platform_packages, |
| lockfiles=default_lockfiles, |
| tag='git_revision') |
| |
| yield (api.test('default') + default_properties + api.step_data( |
| 'cipd describe chromium/fuchsia/fidl', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/fidl', |
| version='GDGGW7Xs89z2apGaYf1mDvbQuHfYIoPexfedNzvKodUC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| 'version:70.0.3538.29', |
| ])) + api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-amd64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-amd64', |
| version='r2S5xldLzzfJa2VzOYgoC6TsIWePSDLBI5FRywd_gHAC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + |
| api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-arm64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-arm64', |
| version='mKI1nni0SW4F1cQAuYnkYU_RtDv47noSKO9vGHJVjzYC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + api.auto_roller.dry_run_step_data() + |
| api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder='chromium-roller'))) |
| |
| yield (api.test('no latest version match') + default_properties + |
| api.step_data( |
| 'cipd describe chromium/fuchsia/fidl', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/fidl', |
| version='GDGGW7Xs89z2apGaYf1mDvbQuHfYIoPexfedNzvKodUC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-amd64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-amd64', |
| version='r2S5xldLzzfJa2VzOYgoC6TsIWePSDLBI5FRywd_gHAC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + |
| api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-arm64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-arm64', |
| version='mKI1nni0SW4F1cQAuYnkYU_RtDv47noSKO9vGHJVjzYC', |
| test_data_tags=[ |
| 'version:70.0.3538.29', |
| ]))) |
| |
| yield (api.test('noop') + default_properties + api.step_data( |
| 'cipd describe chromium/fuchsia/fidl', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/fidl', |
| version='GDGGW7Xs89z2apGaYf1mDvbQuHfYIoPexfedNzvKodUC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-amd64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-amd64', |
| version='r2S5xldLzzfJa2VzOYgoC6TsIWePSDLBI5FRywd_gHAC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + |
| api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-arm64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-arm64', |
| version='mKI1nni0SW4F1cQAuYnkYU_RtDv47noSKO9vGHJVjzYC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + |
| api.step_data('jiri edit', api.json.output({'packages': []}))) |
| |
| yield (api.test('default_with_platform') + platform_pkg_properties + |
| api.step_data( |
| 'cipd describe fuchsia/tools/bb/${platform}', |
| api.cipd.example_describe( |
| package_name='fuchsia/tools/bb/${platform}', |
| version='IX9BXIY7ZejVNabi6zF5lyq2KIW_ABFvBf1u3Cjr2dIC', |
| test_data_tags=[ |
| 'git_revision:aa2dae2ba04148bbe29726c64cfd9d40b6fc2616', |
| ])) + |
| api.step_data( |
| 'cipd describe fuchsia/tools/luci/gsutil/${platform}', |
| api.cipd.example_describe( |
| package_name='fuchsia/tools/luci/gsutil/${platform}', |
| version='IX9BXIY7ZejVNabi6zF5lyq2KIW_ABFvBf1u3Cjr2dIC', |
| test_data_tags=[ |
| 'git_revision:aa2dae2ba04148bbe29726c64cfd9d40b6fc2616', |
| ])) + api.auto_roller.dry_run_step_data() + |
| api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder='tools-roller'))) |
| |
| yield (api.test('default_platform_not_resolved') + platform_pkg_properties + |
| api.step_data( |
| 'cipd describe fuchsia/tools/bb/${platform}', |
| api.cipd.example_describe( |
| package_name='fuchsia/tools/bb/${platform}', |
| version='IX9BXIY7ZejVNabi6zF5lyq2KIW_ABFvBf1u3Cjr2dIC', |
| test_data_tags=[ |
| 'git_revision:aa2dae2ba04148bbe29726c64cfd9d40b6fc2616', |
| ])) + |
| api.step_data( |
| 'cipd describe fuchsia/tools/luci/gsutil/${platform}', |
| api.cipd.example_describe( |
| package_name='fuchsia/tools/luci/gsutil/${platform}', |
| version='IX9BXIY7ZejVNabi6zF5lyq2KIW_ABFvBf1u3Cjr2dIC', |
| test_data_tags=[ |
| 'git_revision:aa2dae2ba04148bbe29726c64cfd9d40b6fc2616', |
| ])) + |
| api.step_data('jiri resolve', api.raw_io.stream_output('')) + |
| api.auto_roller.dry_run_step_data() + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder='tools-roller'))) |
| |
| yield (api.test('with_debug_symbols') + debug_symbols_properties + |
| api.step_data( |
| 'cipd describe chromium/fuchsia/fidl', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/fidl', |
| version='GDGGW7Xs89z2apGaYf1mDvbQuHfYIoPexfedNzvKodUC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| 'version:70.0.3538.29', |
| ])) + api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-amd64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-amd64', |
| version='r2S5xldLzzfJa2VzOYgoC6TsIWePSDLBI5FRywd_gHAC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + |
| api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-arm64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-arm64', |
| version='mKI1nni0SW4F1cQAuYnkYU_RtDv47noSKO9vGHJVjzYC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + |
| api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-debug-symbols-amd64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-debug-symbols-amd64', |
| version='sI1KHrov4_lmsDNnkSJ0MHnKgEtWSl8XRCNRhpAhkacC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + |
| api.step_data( |
| 'cipd describe chromium/fuchsia/webrunner-debug-symbols-arm64', |
| api.cipd.example_describe( |
| package_name='chromium/fuchsia/webrunner-debug-symbols-arm64', |
| version='0soOR1_LGKIyPtlcA2mtD9dHQoKd0K1kkusVwcXLNLAC', |
| test_data_tags=[ |
| 'version:70.0.3538.30', |
| ])) + api.jiri.read_manifest_element( |
| api, |
| 'chromium/chromium', |
| 'package', |
| 'chromium/fuchsia/webrunner-debug-symbols-amd64', |
| test_output={ |
| 'path': 'prebuilt/build_ids/amd64/webrunner', |
| 'attributes': 'debug-symbols,debug-symbols-amd64', |
| }, |
| nesting='fetch and upload debug symbols') + |
| api.jiri.read_manifest_element( |
| api, |
| 'chromium/chromium', |
| 'package', |
| 'chromium/fuchsia/webrunner-debug-symbols-arm64', |
| test_output={ |
| 'path': 'prebuilt/build_ids/arm64/webrunner', |
| 'attributes': 'debug-symbols,debug-symbols-amd64', |
| }, |
| nesting='fetch and upload debug symbols') + |
| api.jiri.read_manifest_element( |
| api, |
| 'chromium/chromium', |
| 'package', |
| 'chromium/fuchsia/webrunner-arm64', |
| test_output={ |
| 'path': 'chromium/fuchsia/arm64/webrunner', |
| }, |
| nesting='fetch and upload debug symbols') + |
| api.jiri.read_manifest_element( |
| api, |
| 'chromium/chromium', |
| 'package', |
| 'chromium/fuchsia/webrunner-amd64', |
| test_output={ |
| 'path': 'chromium/fuchsia/amd64/webrunner', |
| }, |
| nesting='fetch and upload debug symbols') + |
| api.jiri.read_manifest_element( |
| api, |
| 'chromium/chromium', |
| 'package', |
| 'chromium/fuchsia/fidl', |
| test_output={ |
| 'path': 'chromium/fuchsia/fidl', |
| }, |
| nesting='fetch and upload debug symbols') + |
| api.auto_roller.success_step_data() + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder='chromium-roller'))) |
