| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling flutter and dependencies. |
| |
| Flutter and dependencies need to get rolled based on the version |
| dependencies of another fuchsia prebuilt. In this recipe we are |
| calling the fuchsia prebuilt dictating the dependencies "orchestrator". |
| |
| The orchestrator cipd package contains a json file with the dependencies |
| it was built with. The following is an example of the json file: |
| |
| { |
| "flutter_version": "flutter_version_xyz", |
| "engine_version": "engine_version_xyz", |
| "skia_version": "skia_version_xyz", |
| "dart_version": "dart_version_xyz" |
| } |
| |
| This recipe checks if there is a new version of the orchestrator and |
| if it requires different versions of the dependencies from the ones in |
| the source tree. If both conditions are true then it proceeds to roll |
| new versions of projects and packages as described in the |
| rolling_packages_projects parameter and using the versions from the |
| orchestrator dependencies json file. |
| |
| Warning: If this recipe runs in dry mode it will still roll changes into |
| fuchsia dart 3p repository but the fuchsia tree won't pin to the latest |
| version. This is safe as the fuchsia dart 3p hash will be bumped only on a |
| successful roll of all the other dependencies. |
| """ |
| |
| import copy |
| import os |
| |
| from recipe_engine.config import List |
| from recipe_engine.post_process import StepSuccess |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| 'fuchsia/jiri', |
| 'fuchsia/auto_roller', |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/cipd_dependencies', |
| 'fuchsia/dart_util', |
| 'fuchsia/debug_symbols', |
| 'fuchsia/gitiles', |
| 'recipe_engine/archive', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/properties', |
| 'recipe_engine/path', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = { |
| 'project': |
| Property(kind=str, help='Jiri remote manifest project', default=None), |
| 'manifests': |
| Property( |
| kind=List(dict), |
| help=( |
| 'A list of dictionaries with project, manifest and remote keys ' |
| 'used to signal jiri wich manifests to synchronize. ' |
| 'E.g. [{"project": "integration", "manifest": "other/dependency", ' |
| ' "remote": "sso://fuchsia/integration", ' |
| ' "lock_file": "integration/jiri.lock"}]')), |
| 'locks': |
| Property( |
| kind=List(dict), |
| help=('A list of dictionaries with manifest and files keys used to ' |
| 'signal jiri which lock files to update. ' |
| ' E.g. [{"manifest": "integration/fuchsia/flower", ' |
| ' "file": "integration/fuchsia/jiri.lock"}]')), |
| 'orchestrator_package': |
| Property( |
| kind=str, |
| help=('A string with the CIPD package used to validate ' |
| 'flutter and dependencies versions. This is a special ' |
| 'cipd package that contains a json file describing ' |
| 'the dependencies it was built with.')), |
| 'orchestrator_ref': |
| Property( |
| kind=str, |
| default='latest', |
| help=('A cipd ref used to get a given version of the orchestrator ' |
| 'package')), |
| 'orchestrator_import_in': |
| Property( |
| kind=str, |
| default='', |
| help=('A string with the location where the orchestrator package ' |
| 'should be imported into the source tree.')), |
| 'owners': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('The owners responsible for watching this roller ' |
| '(example: "username@google.com").')), |
| 'rolling_packages_projects': |
| Property( |
| kind=List(dict), |
| default=(), |
| help=('A list of dictionaries with rolling package/project, type,' |
| ' and location to import_in. This is used to list the ' |
| 'dependencies that should be updated: ' |
| 'E.g. [{"name": "dart", "type": "package", ' |
| ' "import_in": "fuchsia/prebuilts", ' |
| ' "version_tag": "dart_version"}]')), |
| 'versions_file': |
| Property( |
| kind=str, |
| help=('A string with the path to the versions file inside' |
| 'the orchestrator CIPD package.')), |
| 'validate_against_package': |
| Property( |
| kind=dict, |
| help=('A dictionary with the package and tag for the version to use' |
| ' for validation. A single dependency is used to validate if' |
| ' a roll is needed. E.g. {"package": "flutter/fuchsia", ' |
| ' "version_tag": "flutter_version"} ' |
| 'will validate the source tree version of flutter/fuchsia ' |
| 'with the value of flutter_version from the orchestrator ' |
| 'json file.')), |
| 'dry_run': |
| Property( |
| kind=bool, |
| default=False, |
| help=('Whether to dry-run the auto-roller (CQ+1 and abandon the ' |
| 'change)')), |
| 'debug_symbol_packages': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('A list of strings with the cipd packages containing debug ' |
| 'symbols and their associated packages.')), |
| 'debug_symbol_attribute': |
| Property( |
| kind=str, |
| default='', |
| help=( |
| 'String with the cipd attribute to validate if a cipd package ' |
| 'contains debug symbols.')), |
| 'debug_symbol_import_in': |
| Property( |
| kind=str, |
| default='', |
| help=('String with the path where debug symbols will be imported ' |
| 'in the source tree.')), |
| 'debug_symbol_gcs_buckets': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('List of strings with the gcs buckets where the debug ' |
| 'symbols will be uploaded to.')), |
| 'debug_symbol_remote': |
| Property( |
| kind=str, |
| default='', |
| help='String with the url of the debug symbol remote repository') |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {roller} packages and projects: |
| |
| {packages} |
| |
| Test: CQ |
| CQ-Do-Not-Cancel-Tryjobs: true""" |
| |
| |
| def RunSteps(api, project, manifests, locks, orchestrator_ref, owners, |
| versions_file, rolling_packages_projects, orchestrator_package, |
| orchestrator_import_in, validate_against_package, dry_run, |
| debug_symbol_packages, debug_symbol_attribute, |
| debug_symbol_gcs_buckets, debug_symbol_import_in, |
| debug_symbol_remote): |
| """Run the recipe steps.""" |
| with api.context(infra_steps=True): |
| if owners: |
| with api.step.nest('owners') as presentation: |
| presentation.step_summary_text = ', '.join(owners) |
| |
| # Import manifests |
| with api.step.nest('jiri-manifests') as presentation: |
| api.jiri.init(use_lock_file=True) |
| presentation.logs['manifests'] = str(manifests) |
| for manifest_config in manifests: |
| api.jiri.import_manifest( |
| manifest=manifest_config['manifest'], |
| remote=manifest_config['remote'], |
| name=manifest_config['project']) |
| api.jiri.update(run_hooks=False) |
| api.jiri.run_hooks() |
| |
| deps_info = api.jiri.package([orchestrator_package]) |
| cipd_description = api.cipd.describe(orchestrator_package, orchestrator_ref) |
| orchestrator_current_version = deps_info.json.output[0]['version'] |
| # Is there a new version to roll? |
| orchestrator_expected_version = cipd_description.tags[0].tag |
| if orchestrator_expected_version == orchestrator_current_version: |
| msg = '%s in tree version is [%s] and latest from cipd is [%s]' % ( |
| orchestrator_package, orchestrator_current_version, |
| orchestrator_expected_version) |
| api.step('manifest up-to-date; nothing to roll', |
| None).presentation.step_text = msg |
| return |
| |
| # Read main package versions file. |
| orchestrator_version_dict = api.cipd_dependencies.get_dependencies( |
| 'read versions', orchestrator_package, cipd_description.pin.instance_id, |
| versions_file) |
| |
| # Use version of the validation package to decide whether a roll is needed or |
| # not. |
| validation_info = api.jiri.package([validate_against_package['package']]) |
| local_version = validation_info.json.output[0]['version'] |
| expected_version = 'git_revision:%s' % orchestrator_version_dict[ |
| validate_against_package['version_tag']] |
| if local_version == expected_version: |
| msg = 'Expected version [%s] == local fuchsia tree version [%s]' % ( |
| expected_version, local_version) |
| api.step('validation_summary', cmd=None).presentation.step_text = msg |
| return |
| # Start rolling orchestrator |
| rolling_packages_projects.append({ |
| 'type': 'package', |
| 'name': orchestrator_package, |
| 'version_tag': orchestrator_package, |
| 'import_in': orchestrator_import_in |
| }) |
| orchestrator_version_dict[ |
| orchestrator_package] = orchestrator_expected_version |
| # Roll all the dependencies |
| package_msgs = [] |
| package_msg_template = '%s %s rolled from %s to %s' |
| with api.step.nest('edit jiri manifests'): |
| for package_project in rolling_packages_projects: |
| package_msg = '' |
| if package_project['type'] == 'package': |
| if package_project['name'] == orchestrator_package: |
| expected_version = orchestrator_version_dict[ |
| package_project['version_tag']] |
| else: |
| expected_version = 'git_revision:%s' % orchestrator_version_dict[ |
| package_project['version_tag']] |
| changes = api.jiri.edit_manifest( |
| package_project['import_in'], |
| packages=[(package_project['name'], expected_version)], |
| name=package_project['name']) |
| # Sometimes only a subset of packages are updated. In those cases |
| # the changes list is empty. |
| if changes['packages']: |
| old_version = changes['packages'][0]['old_version'] |
| package_msg = package_msg_template % ( |
| 'package', package_project['name'], old_version, expected_version) |
| else: |
| expected_version = orchestrator_version_dict[ |
| package_project['version_tag']] |
| # We need to save dart/sdk version to update the dart |
| # plugins version later in this recipe. |
| if package_project['name'] == 'dart/sdk': |
| dart_sdk_revision = expected_version |
| changes = api.jiri.edit_manifest( |
| package_project['import_in'], |
| name=package_project['name'], |
| projects=[(package_project['name'], expected_version)]) |
| # Sometimes only a subset of projects are updated. In those cases |
| # the changes list is empty. |
| if changes['projects']: |
| old_version = changes['projects'][0]['old_revision'] |
| package_msg = package_msg_template % ( |
| 'project', package_project['name'], old_version, expected_version) |
| if package_msg: |
| package_msgs.append(package_msg) |
| |
| # Roll dart internal plugins |
| sandbox_dir = api.path.mkdtemp('sandbox-flutter-dart') |
| dart_path = sandbox_dir.join('dart') |
| with api.step.nest('dart sdk checkout'): |
| api.dart_util.checkout(path=dart_path, revision=dart_sdk_revision) |
| |
| with api.step.nest('dart third-party packages'): |
| checkout_root = api.path['start_dir'].join('integration') |
| api.dart_util.update_pkg_manifest( |
| path=dart_path, checkout_root=checkout_root) |
| |
| # Update fuchsia's third-party dart packages, not to be confused with |
| # the previous updating of dart's own third-party packages. |
| manifest_repo = api.path['start_dir'].join('integration') |
| manifest_path = manifest_repo.join('fuchsia', 'topaz', 'topaz') |
| with api.step.nest('third-party dart packages'): |
| checkout_root = api.path['start_dir'] |
| rolled_hash = api.dart_util.update_3p_packages(checkout_root=checkout_root) |
| changes = api.jiri.edit_manifest( |
| 'integration/fuchsia/topaz/dart', |
| name='third_party/dart-pkg', |
| projects=[('third_party/dart-pkg', rolled_hash)]) |
| |
| # Update lock files |
| with api.step.nest('update jiri lock files'): |
| for lock in locks: |
| api.jiri.resolve( |
| local_manifest=True, |
| output=lock['file'], |
| manifests=[lock['manifest']]) |
| |
| roller_name = api.buildbucket.builder_name |
| if roller_name.endswith('-roller'): |
| roller_name = api.buildbucket.builder_name[:7] |
| message = COMMIT_MESSAGE.format( |
| roller=roller_name, |
| packages='\n\n'.join(package_msgs), |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| |
| # All the changes are happening inside one of the git cloned projects. |
| # The "project" property points to the location of the project containing |
| # the changes and we need to move to that directory for the roller to pick |
| # up the changes. |
| project_dir = api.path['start_dir'].join(*project.split('/')) |
| rolled = False |
| with api.context(cwd=project_dir): |
| rolled = api.auto_roller.attempt_roll( |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| ) |
| |
| # Push debug symbols. |
| # If running on dry_run mode skip building and pushing debug symbols |
| # because the correct dependencies won't be available in the tree. |
| if rolled and debug_symbol_gcs_buckets and not dry_run: |
| checkout_root = api.path['start_dir'] |
| api.debug_symbols.fetch_and_upload( |
| project=project, |
| checkout_root=checkout_root, |
| import_in=debug_symbol_import_in, |
| remote=debug_symbol_remote, |
| project_dir=project_dir, |
| packages=debug_symbol_packages, |
| debug_symbol_attribute=debug_symbol_attribute, |
| debug_symbol_gcs_buckets=debug_symbol_gcs_buckets) |
| |
| |
| def GenTests(api): |
| """Tests for cipd with dependents roller.""" |
| properties = { |
| 'project': 'integration', |
| 'manifests': [{ |
| 'project': 'integration', |
| 'manifest': 'other/dependency', |
| 'remote': 'sso://fuchsia/integration', |
| 'lock_file': 'integration/jiri.lock' |
| }, { |
| 'project': 'integration', |
| 'manifest': 'fuchsia/flower', |
| 'remote': 'sso://fuchsia/integration', |
| 'lock_file': 'integration/fuchsia/jiri.lock' |
| }], |
| 'locks': [{ |
| 'manifest': 'a/b', |
| 'file': 'a/b/jiri.lock' |
| }], |
| 'orchestrator_package': 'fuchsia/orchestrator', |
| 'orchestrator_import_in': 'fuchsia/prebuilts', |
| 'validate_against_package': { |
| 'package': 'flutter/fuchsia', |
| 'version_tag': 'flutter_version' |
| }, |
| 'rolling_packages_projects': [ |
| { |
| 'name': 'dart/sdk', |
| 'type': 'project', |
| 'import_in': 'fuchsia/prebuilts', |
| 'version_tag': 'dart_version' |
| }, |
| { |
| 'name': 'skia', |
| 'type': 'project', |
| 'import_in': 'fuchsia/prebuilts', |
| 'version_tag': 'skia_version' |
| }, |
| { |
| 'name': 'flutter', |
| 'type': 'package', |
| 'import_in': 'fuchsia/prebuilts', |
| 'version_tag': 'engine_version' |
| }, |
| ], |
| 'versions_file': 'flutter/versions.json', |
| 'owners': ['abc@gmail.com'], |
| 'dry_run': True, |
| 'debug_symbol_packages': ['flutter/fuchsia-debug-symbols-x64',], |
| 'debug_symbol_attribute': 'debug-symbols', |
| 'debug_symbol_gcs_buckets': ['fuchsia-debug-symbols-shortlived'], |
| 'debug_symbol_import_in': 'integration/fuchsia/prebuilts', |
| 'debug_symbol_remote': 'sso://fuchsia/integration' |
| } |
| |
| package_no_match_test_data = api.step_data( |
| 'cipd describe fuchsia/orchestrator', |
| api.cipd.example_describe( |
| package_name='fuchsia/orchestrator', |
| version='version_abc', |
| test_data_tags=['git_revision:revision_abc'])) |
| jiri_package_test_data = [ |
| { |
| 'name': 'fuchsia/orchestrator', |
| 'path': 'fuchsia/orchestrator', |
| 'version': 'git_revision:revision_abc', |
| 'manifest': 'manifest1' |
| }, |
| ] |
| default_properties = api.properties(**properties) |
| # Version of the cipd orchestrator package is the same as the one |
| # in the tree. |
| yield ( |
| api.test('nothing_to_roll') + default_properties + |
| package_no_match_test_data + |
| api.step_data('jiri package', api.jiri.package(jiri_package_test_data))) |
| |
| # Version of the cipd orchestrator package is different from the one |
| # in the tree and ready to get rolled but flutter version is the same as the |
| # one in the tree. |
| package_match_test_data = api.step_data( |
| 'cipd describe fuchsia/orchestrator', |
| api.cipd.example_describe( |
| package_name='fuchsia/orchestrator', |
| version='version_abc', |
| test_data_tags=['git_revision:revision_jkl'])) |
| package_versions = { |
| 'flutter_version': 'flutter_version_xyz', |
| 'engine_version': 'engine_version_xyz', |
| 'skia_version': 'skia_version_xyz', |
| 'dart_version': 'dart_version_xyz' |
| } |
| jiri_flutter_package_test_data = [ |
| { |
| 'name': 'flutter/fuchsia', |
| 'path': 'flutter/fuchsia', |
| 'version': 'git_revision:flutter_version_xyz', |
| 'manifest': 'manifest1' |
| }, |
| ] |
| yield (api.test('orchestrator_dep_version_same_as_tree') + |
| default_properties + package_match_test_data + api.step_data( |
| 'jiri package', api.jiri.package(jiri_package_test_data)) + |
| api.step_data('read versions', |
| api.file.read_json(json_content=package_versions)) + |
| package_match_test_data + |
| api.step_data('jiri package (2)', |
| api.jiri.package(jiri_flutter_package_test_data)) + |
| api.post_process(StepSuccess, 'validation_summary')) |
| |
| # Version of the validation cipd package is different from the one in the tree |
| # and ready to get rolled along its dependencies. |
| jiri_flutter_package_test_data_no_match = [ |
| { |
| 'name': 'flutter/fuchsia', |
| 'path': 'flutter/fuchsia', |
| 'version': 'git_revision:flutter_version_xyz1', |
| 'manifest': 'manifest1' |
| }, |
| ] |
| yield ( |
| api.test('orchestrator_dep_version_different_from_tree_dry_run') + |
| default_properties + package_match_test_data + |
| api.step_data('jiri package', api.jiri.package(jiri_package_test_data)) + |
| api.step_data('read versions', |
| api.file.read_json(json_content=package_versions)) + |
| api.step_data('jiri package (2)', |
| api.jiri.package(jiri_flutter_package_test_data_no_match)) + |
| api.auto_roller.success_step_data() + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder='flutter-dependents-roller')) |
| + api.auto_roller.success_step_data( |
| name='third-party dart packages.check for completion.check if done (0)' |
| )) |
| |
| # Validate the build and upload debug symbols path. |
| props = copy.deepcopy(properties) |
| props['dry_run'] = False |
| no_dry_run_props = api.properties(**props) |
| yield ( |
| api.test('orchestrator_dep_version_different_from_tree') + |
| no_dry_run_props + package_match_test_data + |
| api.step_data('jiri package', api.jiri.package(jiri_package_test_data)) + |
| api.step_data('read versions', |
| api.file.read_json(json_content=package_versions)) + |
| api.step_data('jiri package (2)', |
| api.jiri.package(jiri_flutter_package_test_data_no_match)) + |
| api.auto_roller.success_step_data() + api.buildbucket.build( |
| api.buildbucket.ci_build_message(builder='flutter-dependents-roller')) |
| + api.auto_roller.success_step_data( |
| name='third-party dart packages.check for completion.check if done (0)' |
| ) + api.jiri.read_manifest_element( |
| api, |
| 'fuchsia/prebuilts', |
| 'package', |
| 'flutter/fuchsia-debug-symbols-x64', |
| test_output={ |
| 'path': 'prebuilt/build_ids/arm64/flutter', |
| 'attributes': 'debug-symbols', |
| })) |
