| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for automatically updating Flutter, flutter/engine, Skia and Dart.""" |
| |
| from collections import OrderedDict |
| |
| import os |
| import re |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/git', |
| 'fuchsia/gitiles', |
| 'fuchsia/jiri', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/python', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = { |
| 'manifest': |
| Property(kind=str, help='Jiri manifest to use', default='topaz/topaz'), |
| 'remote': |
| Property( |
| kind=str, |
| help='Remote manifest repository', |
| default='https://fuchsia.googlesource.com/integration'), |
| 'flutter_manifest': |
| Property( |
| kind=str, help='Jiri manifest for flutter', |
| default='topaz/flutter'), |
| 'dart_manifest': |
| Property(kind=str, help='Jiri manifest for dart', default='topaz/dart'), |
| 'dart_third_party_pkg_manifest': |
| Property( |
| kind=str, |
| help='Jiri manifest for dart_third_party_pkg', |
| default='topaz/dart_third_party_pkg'), |
| 'revision': |
| Property(kind=str, help='flutter/flutter revision', default=''), |
| 'skia_manifest': |
| Property(kind=str, help='Jiri manifest for skia', default='topaz/skia'), |
| 'prebuilt_manifest': |
| Property( |
| kind=str, |
| help='Jiri manifest for prebuilt packages', |
| default='fuchsia/prebuilts'), |
| 'lockfiles': |
| Property( |
| kind=List(str), |
| default=[], |
| help=('The list of lockfiles to update in "${manifest}=${lockfile}"' |
| ' format')), |
| # TODO: delete this property after we have full lockfile support in |
| # integration repo |
| 'enforce_locks': |
| Property( |
| kind=bool, |
| default=False, |
| help='Whether to enforce locks from lockfiles'), |
| } |
| |
| FLUTTER_NAME = 'external/github.com/flutter/flutter' |
| ENGINE_NAME = 'external/github.com/flutter/engine' |
| DART_SDK_NAME = 'dart/sdk' |
| DART_SDK_PKG_NAME = 'fuchsia/dart-sdk/${platform}' |
| FLUTTER_CIPD_PACKAGES = [ |
| 'flutter/fuchsia', 'flutter/fuchsia-debug-symbols-x64', |
| 'flutter/fuchsia-debug-symbols-arm64' |
| ] |
| SKIA_NAME = 'external/skia.googlesource.com/skia' |
| THIRD_PARTY_DART_PKG_NAME = 'third_party/dart-pkg' |
| THIRD_PARTY_DART_PKG_PATH = ['third_party', 'dart-pkg', 'pub'] |
| |
| COMMIT_SUBJECT = """\ |
| [roll] Update {deps} |
| |
| {logs} |
| |
| CQ-Do-Not-Cancel-Tryjobs: true""" |
| |
| LOCAL_IMPORT_FORMAT = """\ |
| <manifest> |
| <imports> |
| <localimport file="{local_manifest}"/> |
| </imports> |
| </manifest> |
| """ |
| |
| THIRD_PARTY_DART_PKG_COMMIT_MSG = """\ |
| [roll] Update third-party dart packages |
| |
| Updated: |
| {updated} |
| """ |
| |
| |
| def ExtractDependencyVersionFromDEPS(api, deps_path, dependency): |
| """Extracts the dependency version from a flutter/engine's DEPS file. |
| |
| Args: |
| api (RecipeApi): Recipe API object. |
| deps_path (Path): A path to a DEPS file. |
| manifest_path (Path): A path to the Jiri manifest to overwrite. |
| dependency (str): The name of a dependency to extract from the DEPS file. |
| """ |
| dependency_revision = '%s_revision' % dependency |
| contents = api.file.read_text( |
| name='read DEPS file for %s' % dependency, |
| source=deps_path, |
| test_data="'%s': 'abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde'" % |
| dependency_revision, |
| ) |
| m = re.search(r"'%s':\s*'(?P<revision>[0-9a-f]{40})'" % dependency_revision, |
| contents) |
| if not m: |
| raise api.step.InfraFailure('failed to find %s in DEPS' % |
| dependency_revision) |
| return m.group('revision') |
| |
| |
| def UpdatePkgManifest(api, dart_path, manifest_path): |
| """Overwrites a dart third party package manifest. |
| |
| Args: |
| api (RecipeApi): Recipe API object. |
| dart_path (Path): A path to the dart/sdk repository. |
| manifest_path (Path): A path to the Jiri manifest to overwrite. |
| """ |
| api.python( |
| name='update %s' % api.path.basename(manifest_path), |
| script=dart_path.join('tools', 'create_pkg_manifest.py'), |
| args=['-d', dart_path.join('DEPS'), '-o', manifest_path], |
| ) |
| |
| |
| def RollChanges(api, deps): |
| """Rolls manifest changes in a git repository. |
| |
| Args: |
| deps (dict[str]str): A map of dependencies that were updated to |
| a log string, summarizing the update. |
| """ |
| # Generate the commit message. |
| commit_message = COMMIT_SUBJECT.format( |
| deps=', '.join(deps.keys()), logs='\n'.join(deps.itervalues())) |
| |
| # Land the changes. |
| api.auto_roller.attempt_roll( |
| gerrit_project='integration', |
| repo_dir=api.path['start_dir'].join('integration'), |
| commit_message=commit_message, |
| ) |
| |
| |
| def Update3pDartPackages(api, jiri_manifest, local_manifest): |
| """Updates fuchsia's third-party packages. |
| |
| This is a very stateful operation: in particular, it overwrites a |
| .jiri_manifest, temporarily stashes the existing manifest changes, updates per |
| the changes made in a local manifest repo, and pushes a change from |
| //third_party/dart-pkg directly to its origin/master. |
| |
| Args: |
| jiri_manifest (Path): The path to a .jiri_manifest. |
| local_manifest (str): The relative path to a local manifest from the jiri |
| root. |
| """ |
| api.file.write_raw( |
| 'overwrite jiri manifest', |
| jiri_manifest, |
| LOCAL_IMPORT_FORMAT.format(local_manifest=local_manifest), |
| ) |
| # Stash the manifest changes while we are updating to prevent overwriting. |
| manifest_repo = api.path['start_dir'].join('integration') |
| with api.context(cwd=manifest_repo): |
| api.git('stash', 'push', '--include-untracked') |
| api.jiri.update(fetch_packages=True, gc=True) |
| api.git('stash', 'pop') |
| |
| third_party_dart_pkg_repo = api.path['start_dir'].join( |
| *THIRD_PARTY_DART_PKG_PATH) |
| with api.context(cwd=third_party_dart_pkg_repo): |
| # Make sure third_party/dart-pkg is at origin/master before running the |
| # update script to catch any manual commits that extend past the revision at |
| # integration's HEAD. |
| api.step('fetch', ['git', 'fetch', 'origin']) |
| api.step('checkout', ['git', 'checkout', 'origin/master']) |
| |
| api.python( |
| 'update', |
| api.path['start_dir'].join('scripts', 'dart', 'update_3p_packages.py'), |
| args=['--debug'], |
| ) |
| |
| changed_files = api.git( |
| 'ls-files', |
| 'modified', |
| '--deleted', |
| '--others', |
| '--exclude-standard', |
| stdout=api.raw_io.output(), |
| ).stdout |
| |
| commit_msg = THIRD_PARTY_DART_PKG_COMMIT_MSG.format(updated=changed_files) |
| commit_step = api.git.commit( |
| message=commit_msg, |
| all_files=True, |
| ok_ret='any', |
| ) |
| commit_step.presentation.logs['commit message'] = commit_msg.splitlines() |
| |
| # Since we fetch first and since we are now at or ahead of origin/master, |
| # git push is at worst a zero-returning no-op, so cannot be a failing step. |
| api.step('fetch', ['git', 'fetch', 'origin']) |
| api.git.push(ref='HEAD:master') |
| |
| revision = api.git.get_hash() |
| api.step.active_result.presentation.logs['revision'] = [revision] |
| return revision |
| |
| |
| def UpdateDeps(api, updated_deps, manifest_path, flutter_revision, |
| flutter_manifest_path, dart_manifest_path, |
| dart_third_party_pkg_manifest_path, skia_manifest_path, |
| prebuilt_manifest_path): |
| """Updates dart- and flutter-related dependencies to be reflected in the roll. |
| |
| This method makes local changes to a checked out manifest repo. |
| |
| Args: |
| updated_deps (OrderedDict): A dictionary mapping project name to a |
| formatted log of the corresponding changes. |
| manifest_path (Path): The path to the main checked out manifest. |
| flutter_revision (str): The revision at which to checkout out |
| flutter/flutter. |
| flutter_manifest_path (Path): The path to the flutter manifest. |
| dart_manifest_path (Path): The path to the dart manifest. |
| dart_third_party_pkg_manifest_path (Path): The path to the manifest of |
| dart's third-party dependencies. |
| prebuilt_manifest_path (Path): The path to the prebuilt manifest. |
| """ |
| # Set up a temporary sandbox directory to do the required manipulations to |
| # roll flutter, flutter engine, and dart. |
| sandbox_dir = api.path.mkdtemp('sandbox-flutter-dart') |
| # Attempt to update the manifest with a new flutter revision. |
| with api.step.nest('flutter/flutter'): |
| flutter_remote = api.auto_roller.update_manifest_project( |
| manifest=flutter_manifest_path, |
| project_name=FLUTTER_NAME, |
| revision=flutter_revision, |
| updated_deps=updated_deps, |
| ) |
| if not updated_deps: |
| return |
| |
| # Get the flutter/flutter dependency on flutter/engine. |
| flutter_path = sandbox_dir.join('flutter') |
| api.git.checkout( |
| url=flutter_remote, |
| path=flutter_path, |
| ref=flutter_revision, |
| ) |
| engine_revision = api.file.read_text( |
| name='read flutter engine version', |
| source=flutter_path.join('bin', 'internal', 'engine.version'), |
| test_data='xyz000', |
| ).strip() |
| |
| # Attempt to update the manifest with a new engine revision. |
| with api.step.nest('flutter/engine'): |
| engine_remote = api.auto_roller.update_manifest_project( |
| manifest=flutter_manifest_path, |
| project_name=ENGINE_NAME, |
| revision=engine_revision, |
| updated_deps=updated_deps, |
| ) |
| |
| # Get the flutter/engine dependency on Dart & Skia. |
| engine_path = sandbox_dir.join('engine') |
| api.git.checkout( |
| url=engine_remote, |
| path=engine_path, |
| ref=engine_revision, |
| ) |
| |
| # Update flutter prebuilts |
| for prebuilt in FLUTTER_CIPD_PACKAGES: |
| api.jiri.edit_manifest( |
| manifest=prebuilt_manifest_path, |
| packages=[(prebuilt, 'git_revision:%s' % engine_revision)]) |
| |
| with api.step.nest('skia'): |
| skia_revision = ExtractDependencyVersionFromDEPS(api, |
| engine_path.join('DEPS'), |
| 'skia') |
| api.auto_roller.update_manifest_project( |
| manifest=skia_manifest_path, |
| project_name=SKIA_NAME, |
| revision=skia_revision, |
| updated_deps=updated_deps, |
| ) |
| |
| with api.step.nest('dart sdk'): |
| dart_revision = ExtractDependencyVersionFromDEPS(api, |
| engine_path.join('DEPS'), |
| 'dart') |
| dart_remote = api.auto_roller.update_manifest_project( |
| manifest=dart_manifest_path, |
| project_name=DART_SDK_NAME, |
| revision=dart_revision, |
| updated_deps=updated_deps, |
| ) |
| |
| # Update prebuilt for dart/sdk |
| api.jiri.edit_manifest( |
| manifest=prebuilt_manifest_path, |
| packages=[(DART_SDK_PKG_NAME, 'git_revision:' + dart_revision)]) |
| |
| # Get dart/sdk. |
| dart_path = sandbox_dir.join('dart') |
| api.git.checkout( |
| url=dart_remote, |
| path=dart_path, |
| ref=dart_revision, |
| ) |
| |
| # Update the package manifests. |
| with api.step.nest('dart third-party packages'): |
| UpdatePkgManifest( |
| api, |
| dart_path=dart_path, |
| manifest_path=dart_third_party_pkg_manifest_path, |
| ) |
| |
| # Update fuchsia's third-party dart packages, not to be confused with |
| # the previous updating of dart's own third-party packages. |
| with api.step.nest('third-party dart packages'): |
| third_party_dart_pkg_revision = Update3pDartPackages( |
| api=api, |
| jiri_manifest=api.path['start_dir'].join('.jiri_manifest'), |
| local_manifest=os.path.relpath( |
| str(manifest_path), |
| str(api.path['start_dir']), |
| ), |
| ) |
| api.auto_roller.update_manifest_project( |
| manifest=dart_manifest_path, |
| project_name=THIRD_PARTY_DART_PKG_NAME, |
| revision=third_party_dart_pkg_revision, |
| updated_deps=updated_deps, |
| ) |
| |
| |
| def RunSteps(api, manifest, remote, revision, dart_manifest, flutter_manifest, |
| dart_third_party_pkg_manifest, skia_manifest, prebuilt_manifest, |
| lockfiles, enforce_locks): |
| with api.context(infra_steps=True): |
| api.jiri.init(use_lock_file=enforce_locks) |
| api.jiri.import_manifest( |
| manifest=manifest, |
| remote=remote, |
| name='integration', |
| ) |
| api.jiri.update(run_hooks=False) |
| |
| flutter_revision = revision or api.buildbucket.build.input.gitiles_commit.id |
| |
| manifest_repo = api.path['start_dir'].join('integration') |
| manifest_path = manifest_repo.join(manifest) |
| flutter_manifest_path = manifest_repo.join(*flutter_manifest.split('/')) |
| dart_manifest_path = manifest_repo.join(*dart_manifest.split('/')) |
| skia_manifest_path = manifest_repo.join(*skia_manifest.split('/')) |
| dart_third_party_pkg_manifest_path = manifest_repo.join( |
| *dart_third_party_pkg_manifest.split('/')) |
| prebuilt_manifest_path = manifest_repo.join(*prebuilt_manifest.split('/')) |
| |
| updated_deps = OrderedDict() |
| UpdateDeps(api, updated_deps, manifest_path, flutter_revision, |
| flutter_manifest_path, dart_manifest_path, |
| dart_third_party_pkg_manifest_path, skia_manifest_path, |
| prebuilt_manifest_path) |
| # Update lockfiles |
| with api.context(cwd=manifest_repo): |
| for lock_entry in lockfiles: |
| fields = lock_entry.split('=') |
| manifest = fields[0] |
| lock = fields[1] |
| api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest]) |
| if updated_deps: |
| RollChanges(api, updated_deps) |
| |
| |
| def GenTests(api): |
| project_names = { |
| 'flutter/flutter': FLUTTER_NAME, |
| 'flutter/engine': ENGINE_NAME, |
| 'skia': SKIA_NAME, |
| 'dart sdk': DART_SDK_NAME, |
| 'third-party dart packages': THIRD_PARTY_DART_PKG_NAME, |
| } |
| noop_edit = lambda name: api.step_data( |
| '%s.jiri edit %s' % (name, project_names[name]), |
| api.json.output({'projects': []}), |
| ) |
| |
| def log_data(name): |
| return api.gitiles.log('%s.log %s' % (name, project_names[name]), 'A') |
| |
| flutter_check_data = api.jiri.read_manifest_element( |
| api=api, |
| manifest='topaz/flutter', |
| element_name=FLUTTER_NAME, |
| element_type='project', |
| test_output={ |
| 'remote': 'https://fuchsia.googlesource.com/third_party/flutter', |
| }, |
| nesting='flutter/flutter') |
| flutter_log_data = log_data('flutter/flutter') |
| |
| engine_check_data = api.jiri.read_manifest_element( |
| api=api, |
| manifest='topaz/flutter', |
| element_name=ENGINE_NAME, |
| element_type='project', |
| test_output={ |
| 'remote': 'https://fuchsia.googlesource.com/third_party/flutter', |
| }, |
| nesting='flutter/engine') |
| engine_log_data = log_data('flutter/engine') |
| |
| dart_sdk_check_data = api.jiri.read_manifest_element( |
| api=api, |
| manifest='topaz/dart', |
| element_name=DART_SDK_NAME, |
| element_type='project', |
| test_output={ |
| 'remote': 'https://fuchsia.googlesource.com/third_party/dart', |
| }, |
| nesting='dart sdk') |
| dart_sdk_log_data = log_data('dart sdk') |
| |
| skia_check_data = api.jiri.read_manifest_element( |
| api=api, |
| manifest='topaz/skia', |
| element_name=SKIA_NAME, |
| element_type='project', |
| test_output={ |
| 'remote': 'https://fuchsia.googlesource.com/third_party/skia', |
| }, |
| nesting='skia') |
| skia_log_data = log_data('skia') |
| |
| third_party_dart_pkg_check_data = api.jiri.read_manifest_element( |
| api=api, |
| manifest='topaz/dart', |
| element_name=THIRD_PARTY_DART_PKG_NAME, |
| element_type='project', |
| test_output={ |
| 'remote': 'https://fuchsia.googlesource.com/third_party/dart-pkg', |
| }, |
| nesting='third-party dart packages') |
| third_party_dart_pkg_log_data = log_data('third-party dart packages') |
| |
| yield (api.test('noop roll') + api.properties(revision='abc123') + |
| flutter_check_data + noop_edit('flutter/flutter')) |
| |
| yield (api.test('noop roll with lockfile') + api.properties( |
| revision='abc123', lockfiles=['manifest/topaz=jiri.lock']) + |
| flutter_check_data + noop_edit('flutter/flutter')) |
| |
| yield (api.test('flutter/flutter only') + api.properties(revision='abc123') + |
| flutter_check_data + flutter_log_data + engine_check_data + |
| skia_check_data + dart_sdk_check_data + |
| third_party_dart_pkg_check_data + noop_edit('flutter/engine') + |
| noop_edit('skia') + noop_edit('dart sdk') + |
| noop_edit('third-party dart packages') + |
| api.auto_roller.success_step_data()) |
| |
| yield (api.test('flutter/flutter and flutter/engine') + |
| api.properties(revision='abc123') + flutter_check_data + |
| flutter_log_data + engine_check_data + engine_log_data + |
| dart_sdk_check_data + third_party_dart_pkg_check_data + |
| noop_edit('dart sdk') + skia_check_data + noop_edit('skia') + |
| noop_edit('third-party dart packages') + |
| api.auto_roller.success_step_data()) |
| |
| yield (api.test('flutter/flutter, flutter/engine and skia') + |
| api.properties(revision='abc123') + flutter_check_data + |
| flutter_log_data + engine_check_data + engine_log_data + |
| dart_sdk_check_data + noop_edit('dart sdk') + skia_check_data + |
| skia_log_data + third_party_dart_pkg_check_data + |
| noop_edit('third-party dart packages') + |
| api.auto_roller.success_step_data()) |
| |
| yield (api.test('cannot find dart version') + |
| api.properties(revision='abc123') + flutter_check_data + |
| flutter_log_data + engine_check_data + engine_log_data + |
| skia_check_data + skia_log_data + |
| api.step_data('dart sdk.read DEPS file for dart', |
| api.raw_io.output_text('stuff'))) |
| |
| yield (api.test('flutter/flutter, flutter/engine, and dart') + |
| api.properties(revision='abc123') + flutter_check_data + |
| flutter_log_data + engine_check_data + engine_log_data + |
| dart_sdk_check_data + dart_sdk_log_data + skia_check_data + |
| noop_edit('skia') + third_party_dart_pkg_check_data + |
| third_party_dart_pkg_log_data + api.auto_roller.success_step_data()) |
| |
| yield (api.test('flutter/flutter, flutter/engine, skia and dart') + |
| api.properties(revision='abc123') + flutter_check_data + |
| flutter_log_data + engine_check_data + engine_log_data + |
| dart_sdk_check_data + dart_sdk_log_data + skia_check_data + |
| skia_log_data + third_party_dart_pkg_check_data + |
| third_party_dart_pkg_log_data + api.auto_roller.success_step_data()) |
| |
| yield (api.test('no revision') + flutter_check_data + flutter_log_data + |
| engine_check_data + engine_log_data + dart_sdk_check_data + |
| dart_sdk_log_data + skia_check_data + skia_log_data + |
| third_party_dart_pkg_check_data + third_party_dart_pkg_log_data + |
| api.auto_roller.success_step_data() + |
| api.buildbucket.ci_build(revision='321abc')) |
