| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling CIPD prebuilts which depend on other prebuilts. |
| |
| Version dependencies are defined in a file called versions.json on the root |
| folder of the CIPD package. This recipe reads the dependencies and validate |
| the correct versions are already in the tree. If all the dependencies are |
| satisfied then the new CIPD package is rolled if not it will just present |
| a message notifying is waiting for a given version. |
| """ |
| import copy |
| import re |
| |
| from recipe_engine.config import List |
| from recipe_engine.post_process import StatusSuccess |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/cipd_dependencies', |
| 'fuchsia/jiri', |
| 'recipe_engine/archive', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = { |
| 'project': |
| Property(kind=str, help='Jiri remote manifest project', default=None), |
| 'manifests': |
| Property( |
| kind=List(dict), |
| help=( |
| 'A list of dictionaries with project, manifest and remote key.' |
| )), |
| 'import_in': |
| Property( |
| kind=str, help='Path to the manifest to edit relative to $project'), |
| 'package': |
| Property( |
| kind=str, help='The list of CIPD packages to update in $import_in'), |
| 'lockfiles': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('The list of lockfiles to update in "${manifest}=${lockfile}"' |
| 'format')), |
| 'dry_run': |
| Property( |
| kind=bool, |
| default=False, |
| help=('Whether to dry-run the auto-roller (CQ+1 and abandon the ' |
| 'change)')), |
| 'tag': |
| Property( |
| kind=str, |
| default='version', |
| help=('A CIPD tag common to all $packages where a common version ' |
| 'can be extracted')), |
| 'ref': |
| Property( |
| kind=str, |
| default='latest', |
| help='A common CIPD ref to resolve when rolling a set of packages'), |
| 'owners': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('The owners responsible for watching this roller ' |
| '(example: "username@google.com").')), |
| 'package_dependencies': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('List of package dependencies in the form of ' |
| 'package_name=package_key where package_name is the ' |
| 'cipd package name and package_key is the key in ' |
| 'versions file with the expected version for the ' |
| 'dependency cipd package.')), |
| 'project_dependencies': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('List of project dependencies in the form of ' |
| 'project_name=package_key')), |
| 'version_file': |
| Property( |
| kind=str, |
| help=('A string with the path to the versions file inside ' |
| 'the CIPD package.')) |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {roller} CIPD packages: |
| |
| {packages} |
| |
| From: {old_version} |
| To: {version} |
| |
| Test: CQ |
| Cq-Cl-Tag: roller-builder:{builder} |
| Cq-Cl-Tag: roller-bid:{build_id} |
| CQ-Do-Not-Cancel-Tryjobs: true""" |
| |
| CIPD_URL = 'https://chrome-infra-packages.appspot.com/p/{package}/+/{version}' |
| |
| |
| def _get_platform_specific_packages(package, output): |
| platform_regex = '(?<=' + package.replace('${platform}', |
| r'\${platform=).*(?=})') |
| if not package.endswith('${platform}'): |
| return [package] |
| pattern = re.compile(platform_regex) |
| match = pattern.search(output) |
| if match: |
| platforms = match.group(0).split(',') |
| return [package.replace('${platform}', platform) for platform in platforms] |
| |
| |
| def _append_urls(packages, old_version, new_version): |
| package_line = '{package} old:{old} new:{new}' |
| packages_with_urls = [] |
| for package in packages: |
| if '${platform}' in package: |
| packages_with_urls.append(package) |
| else: |
| packages_with_urls.append( |
| package_line.format( |
| old=CIPD_URL.format(package=package, version=old_version), |
| new=CIPD_URL.format(package=package, version=new_version), |
| package=package)) |
| return packages_with_urls |
| |
| |
| def RunSteps(api, manifests, import_in, package, lockfiles, dry_run, tag, ref, |
| owners, package_dependencies, version_file, project_dependencies, |
| project): |
| """Run the recipe steps.""" |
| del (tag) |
| with api.context(infra_steps=True): |
| if owners: |
| with api.step.nest('owners') as owners_presentation: |
| owners_presentation.step_summary_text = ', '.join(owners) |
| |
| api.jiri.init(use_lock_file=True) |
| for manifest_config in manifests: |
| api.jiri.import_manifest( |
| manifest=manifest_config['manifest'], |
| remote=manifest_config['remote'], |
| name=manifest_config['project']) |
| api.jiri.update(run_hooks=False) |
| api.jiri.run_hooks() |
| deps_info = api.jiri.package([package]) |
| cipd_description = api.cipd.describe(package, ref) |
| |
| # Is there a new version to roll? |
| if cipd_description.tags[0].tag == deps_info.json.output[0]['version']: |
| api.step('manifest up-to-date; nothing to roll', None) |
| return |
| |
| version = cipd_description.tags[0].tag |
| # We need to roll the new version. |
| versions_json = api.cipd_dependencies.get_dependencies( |
| 'read versions', package, cipd_description.pin.instance_id, |
| version_file) |
| summary = api.cipd_dependencies.validate_against_tree( |
| package=package, |
| dependencies=package_dependencies, |
| versions_dict=versions_json) |
| if summary: |
| api.step('Package dependencies not satisfied: {}'.format(summary), None) |
| return |
| # Verify source dependencies are satisfied. |
| summary = api.cipd_dependencies.validate_against_tree( |
| package, project_dependencies, versions_json, is_package=False) |
| if summary: |
| api.step('Project dependencies not satisfied: {}'.format(summary), None) |
| return |
| |
| changes = api.jiri.edit_manifest(import_in, packages=[(package, version)]) |
| old_version = changes['packages'][0]['old_version'] |
| exact_packages = set() |
| |
| # Update the lockfiles. |
| for lock_entry in lockfiles: |
| fields = lock_entry.split('=') |
| manifest = fields[0] |
| lock = fields[1] |
| resolve_output = api.jiri.resolve( |
| local_manifest=True, output=lock, manifests=[manifest]).stdout |
| platform_pkgs = _get_platform_specific_packages(package, resolve_output) |
| exact_packages = exact_packages.union(platform_pkgs) |
| exact_packages.add(package) |
| |
| exact_packages = sorted(exact_packages) |
| packages_with_urls = _append_urls(exact_packages, old_version, version) |
| |
| roller_name = api.buildbucket.builder_name.rstrip('-roller') |
| message = COMMIT_MESSAGE.format( |
| roller=roller_name, |
| packages='\n'.join(packages_with_urls), |
| old_version=old_version, |
| version=version, |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| |
| # Land the changes. |
| project_dir = api.path['start_dir'].join(*project.split('/')) |
| with api.context(cwd=project_dir): |
| api.auto_roller.attempt_roll( |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| ) |
| |
| |
| def GenTests(api): |
| properties = { |
| 'project': 'integration', |
| 'manifests': [{ |
| 'project': 'integration', |
| 'manifest': 'other/dependency', |
| 'remote': 'sso://fuchsia/integration' |
| }, { |
| 'project': 'integration', |
| 'manifest': 'fuchsia/flower', |
| 'remote': 'sso://fuchsia/integration' |
| }], |
| 'import_in': 'fuchsia/prebuilts', |
| 'package': 'flutter/dependent/${platform}', |
| 'lockfiles': ['integration/flower=integration/jiri.lock'], |
| 'tag': 'git_revision', |
| 'version_file': 'flutter/versions.json', |
| 'package_dependencies': ['flutter/fuchsia=engine_version'], |
| 'project_dependencies': ['dart/sdk=dart_version'], |
| 'owners': ['abc@google.com'] |
| } |
| flutter_dependent_no_match_test_data = api.step_data( |
| 'cipd describe flutter/dependent/${platform}', |
| api.cipd.example_describe( |
| package_name='flutter/dependent', |
| version='dependent_version_abc', |
| test_data_tags=['git_revision:dependent_revision_jkl'])) |
| package_test_data = [ |
| { |
| 'name': 'flutter/dependent', |
| 'path': 'path/flutter/dependent', |
| 'version': 'git_revision:dependent_revision_jkl', |
| 'manifest': 'manifest1' |
| }, |
| ] |
| default_properties = api.properties(**properties) |
| yield (api.test('nothing_to_roll') + default_properties + |
| flutter_dependent_no_match_test_data + |
| api.step_data('jiri package', api.jiri.package(package_test_data))) |
| |
| flutter_dependent_match_test_data = api.step_data( |
| 'cipd describe flutter/dependent/${platform}', |
| api.cipd.example_describe( |
| package_name='flutter/dependent', |
| version='dependent_version_def', |
| test_data_tags=['git_revision:dependent_revision_lmn'])) |
| versions_json_deps_not_satisfied = { |
| 'engine_version': 'engine_version_xyz', |
| } |
| dependencies_package_test_data = [ |
| { |
| 'name': 'flutter/fuchsia', |
| 'path': 'path/flutter/fuchsia', |
| 'version': 'git_revision:engine_revision_abc', |
| 'manifest': 'manifest1' |
| }, |
| ] |
| yield (api.test('package_dependencies_not_satisfied') + default_properties + |
| flutter_dependent_match_test_data + |
| api.step_data('jiri package', api.jiri.package(package_test_data)) + |
| api.step_data('jiri package (2)', |
| api.jiri.package(dependencies_package_test_data)) + |
| api.step_data( |
| 'read versions', |
| api.file.read_json(json_content=versions_json_deps_not_satisfied))) |
| |
| deps_satisfied = { |
| 'engine_version': 'engine_revision_abc', |
| 'dart_version': 'dart_version_abc' |
| } |
| project_test_data = [{ |
| 'name': 'dart/sdk', |
| 'path': 'path/fuchsia/dart/sdk', |
| 'revision': 'dart_version_abc', |
| 'manifest': 'manifest1' |
| }] |
| properties_no_platform = copy.deepcopy(properties) |
| properties_no_platform['package'] = 'flutter/dependent' |
| default_properties = api.properties(**properties_no_platform) |
| flutter_dependent_match_test_data_no_platform = api.step_data( |
| 'cipd describe flutter/dependent', |
| api.cipd.example_describe( |
| package_name='flutter/dependent', |
| version='dependent_version_def', |
| test_data_tags=['git_revision:dependent_revision_lmn'])) |
| yield ( |
| api.test('dependencies_satisfied_package_with_platform') + |
| default_properties + flutter_dependent_match_test_data_no_platform + |
| api.step_data('jiri package', api.jiri.package(package_test_data)) + |
| api.step_data( |
| 'jiri resolve', |
| stdout=api.jiri.example_resolve_data('flutter/dependent')) + |
| api.step_data('jiri package (2)', |
| api.jiri.package(dependencies_package_test_data)) + |
| api.step_data('jiri project', api.jiri.project(project_test_data)) + |
| api.step_data('read versions', |
| api.file.read_json(json_content=deps_satisfied)) + |
| api.auto_roller.success_step_data() + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder='flutter-dependent-roller'))) |
| |
| default_properties = api.properties(**properties) |
| yield ( |
| api.test('dependencies_satisfied') + default_properties + |
| flutter_dependent_match_test_data + |
| api.step_data('jiri package', api.jiri.package(package_test_data)) + |
| api.step_data( |
| 'jiri resolve', |
| stdout=api.jiri.example_resolve_data('flutter/dependent/${platform}')) |
| + api.step_data('jiri package (2)', |
| api.jiri.package(dependencies_package_test_data)) + |
| api.step_data('jiri project', api.jiri.project(project_test_data)) + |
| api.step_data('read versions', |
| api.file.read_json(json_content=deps_satisfied)) + |
| api.auto_roller.dry_run_step_data() + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder='flutter-dependent-roller'))) |
| project_test_data_not_satisfied = [{ |
| 'name': 'dart/sdk', |
| 'path': 'path/fuchsia/dart/sdk', |
| 'revision': 'dart_version_xyz', |
| 'manifest': 'manifest1' |
| }] |
| yield (api.test('project_dependencies_not_satisfied') + default_properties + |
| flutter_dependent_match_test_data + |
| api.step_data('jiri package', api.jiri.package(package_test_data)) + |
| api.step_data('jiri package (2)', |
| api.jiri.package(dependencies_package_test_data)) + |
| api.step_data('jiri project', |
| api.jiri.project(project_test_data_not_satisfied)) + |
| api.step_data('read versions', |
| api.file.read_json(json_content=deps_satisfied))) |
