blob: f28f7d03e9243e8824f1e88aad86bd23be88eb78 [file] [log] [blame]
# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for automatically updating Flutter and Dart."""
from collections import OrderedDict
import os
import re
from recipe_engine.config import List
from recipe_engine.recipe_api import Property
DEPS = [
'fuchsia/auto_roller',
'fuchsia/build',
'fuchsia/buildbucket_util',
'fuchsia/git',
'fuchsia/gitiles',
'fuchsia/jiri',
'fuchsia/upload_debug_symbols',
'recipe_engine/archive',
'recipe_engine/buildbucket',
'recipe_engine/context',
'recipe_engine/file',
'recipe_engine/json',
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/python',
'recipe_engine/raw_io',
'recipe_engine/step',
]
PROPERTIES = {
'manifest':
Property(kind=str, help='Jiri manifest to use', default='topaz/topaz'),
'remote':
Property(
kind=str,
help='Remote manifest repository',
default='https://fuchsia.googlesource.com/integration'),
'flutter_manifest':
Property(
kind=str, help='Jiri manifest for flutter',
default='topaz/flutter'),
'dart_manifest':
Property(kind=str, help='Jiri manifest for dart', default='topaz/dart'),
'dart_third_party_pkg_manifest':
Property(
kind=str,
help='Jiri manifest for dart_third_party_pkg',
default='topaz/dart_third_party_pkg'),
'revision':
Property(kind=str, help='flutter/flutter revision', default=''),
'prebuilt_manifest':
Property(
kind=str,
help='Jiri manifest for prebuilt packages',
default='fuchsia/prebuilts'),
'lockfiles':
Property(
kind=List(str),
default=[],
help=('The list of lockfiles to update in "${manifest}=${lockfile}"'
' format')),
# TODO: delete this property after we have full lockfile support in
# integration repo
'enforce_locks':
Property(
kind=bool,
default=False,
help='Whether to enforce locks from lockfiles'),
'debug_symbol_attribute':
Property(
kind=str,
default='debug-symbols',
help='Jiri attribute to match debug symbol packages'),
'debug_symbol_gcs_buckets':
Property(
kind=List(str),
default=(),
help='GCS buckets to upload debug symbols upon successful roll'),
}
FLUTTER_NAME = 'external/github.com/flutter/flutter'
ENGINE_REMOTE = 'https://chromium.googlesource.com/external/github.com/flutter/engine'
DART_SDK_NAME = 'dart/sdk'
DART_SDK_PKG_NAME = 'fuchsia/dart-sdk/${platform}'
DEBUG_SYMBOL_PACKAGES = [
'flutter/fuchsia-debug-symbols-x64', 'flutter/fuchsia-debug-symbols-arm64'
]
FLUTTER_CIPD_PACKAGES = ['flutter/fuchsia', 'flutter/sky_engine'
] + DEBUG_SYMBOL_PACKAGES
THIRD_PARTY_DART_PKG_NAME = 'third_party/dart-pkg'
THIRD_PARTY_DART_PKG_PATH = ['third_party', 'dart-pkg', 'pub']
COMMIT_MESSAGE = """\
[roll] Update {deps}
{logs}
Cq-Cl-Tag: roller-builder:{builder}
Cq-Cl-Tag: roller-bid:{build_id}
CQ-Do-Not-Cancel-Tryjobs: true"""
LOCAL_IMPORT_FORMAT = """\
<manifest>
<imports>
<localimport file="{local_manifest}"/>
</imports>
</manifest>
"""
THIRD_PARTY_DART_PKG_COMMIT_MSG = """\
[roll] Update third-party dart packages
Updated:
{updated}
"""
def extract_dependency_version_from_deps(api, deps_path, dependency):
"""Extracts the dependency version from a flutter/engine's DEPS file.
Args:
api (RecipeApi): Recipe API object.
deps_path (Path): A path to a DEPS file.
manifest_path (Path): A path to the Jiri manifest to overwrite.
dependency (str): The name of a dependency to extract from the DEPS file.
"""
dependency_revision = '%s_revision' % dependency
contents = api.file.read_text(
name='read DEPS file for %s' % dependency,
source=deps_path,
test_data="'%s': 'abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde'" %
dependency_revision,
)
m = re.search(r"'%s':\s*'(?P<revision>[0-9a-f]{40})'" % dependency_revision,
contents)
if not m:
raise api.step.InfraFailure('failed to find %s in DEPS' %
dependency_revision)
return m.group('revision')
def update_pkg_manifest(api, dart_path, manifest_path):
"""Overwrites a dart third party package manifest.
Args:
api (RecipeApi): Recipe API object.
dart_path (Path): A path to the dart/sdk repository.
manifest_path (Path): A path to the Jiri manifest to overwrite.
"""
api.python(
name='update %s' % api.path.basename(manifest_path),
script=dart_path.join('tools', 'create_pkg_manifest.py'),
args=['-d', dart_path.join('DEPS'), '-o', manifest_path],
)
def roll_changes(api, deps):
"""Rolls manifest changes in a git repository.
Args:
deps (dict[str]str): A map of dependencies that were updated to
a log string, summarizing the update.
"""
# Generate the commit message.
commit_message = COMMIT_MESSAGE.format(
deps=', '.join(deps.keys()),
logs='\n'.join(deps.itervalues()),
builder=api.buildbucket.builder_name,
build_id=api.buildbucket_util.id,
)
# Land the changes.
return api.auto_roller.attempt_roll(
gerrit_project='integration',
repo_dir=api.path['start_dir'].join('integration'),
commit_message=commit_message,
)
def update_3p_dart_packages(api, jiri_manifest, local_manifest):
"""Updates fuchsia's third-party packages.
This is a very stateful operation: in particular, it overwrites a
.jiri_manifest, temporarily stashes the existing manifest changes, updates per
the changes made in a local manifest repo, and pushes a change from
//third_party/dart-pkg directly to its origin/master.
Args:
jiri_manifest (Path): The path to a .jiri_manifest.
local_manifest (str): The relative path to a local manifest from the jiri
root.
"""
api.file.write_raw(
'overwrite jiri manifest',
jiri_manifest,
LOCAL_IMPORT_FORMAT.format(local_manifest=local_manifest),
)
# Stash the manifest changes while we are updating to prevent overwriting.
manifest_repo = api.path['start_dir'].join('integration')
with api.context(cwd=manifest_repo):
api.git('stash', 'push', '--include-untracked')
api.jiri.update(fetch_packages=True, gc=True)
api.git('stash', 'pop')
third_party_dart_pkg_repo = api.path['start_dir'].join(
*THIRD_PARTY_DART_PKG_PATH)
with api.context(cwd=third_party_dart_pkg_repo):
# Make sure third_party/dart-pkg is at origin/master before running the
# update script to catch any manual commits that extend past the revision at
# integration's HEAD.
api.step('fetch', ['git', 'fetch', 'origin'])
api.step('checkout', ['git', 'checkout', 'origin/master'])
api.python(
'update',
api.path['start_dir'].join('scripts', 'dart', 'update_3p_packages.py'),
args=['--debug'],
)
changed_files = api.git(
'ls-files',
'modified',
'--deleted',
'--others',
'--exclude-standard',
stdout=api.raw_io.output(),
).stdout
commit_msg = THIRD_PARTY_DART_PKG_COMMIT_MSG.format(updated=changed_files)
commit_step = api.git.commit(
message=commit_msg,
all_files=True,
ok_ret='any',
)
commit_step.presentation.logs['commit message'] = commit_msg.splitlines()
# Since we fetch first and since we are now at or ahead of origin/master,
# git push is at worst a zero-returning no-op, so cannot be a failing step.
api.step('fetch', ['git', 'fetch', 'origin'])
api.git.push(ref='HEAD:master')
revision = api.git.get_hash()
api.step.active_result.presentation.logs['revision'] = [revision]
return revision
def update_deps(api, updated_deps, manifest_path, flutter_revision,
flutter_manifest_path, dart_manifest_path,
dart_third_party_pkg_manifest_path, prebuilt_manifest_path):
"""Updates dart- and flutter-related dependencies to be reflected in the roll.
This method makes local changes to a checked out manifest repo.
Args:
updated_deps (OrderedDict): A dictionary mapping project/package name to
a formatted log of the corresponding changes.
manifest_path (Path): The path to the main checked out manifest.
flutter_revision (str): The revision at which to checkout out
flutter/flutter.
flutter_manifest_path (Path): The path to the flutter manifest.
dart_manifest_path (Path): The path to the dart manifest.
dart_third_party_pkg_manifest_path (Path): The path to the manifest of
dart's third-party dependencies.
prebuilt_manifest_path (Path): The path to the prebuilt manifest.
"""
# Set up a temporary sandbox directory to do the required manipulations to
# roll flutter, flutter engine, and dart.
sandbox_dir = api.path.mkdtemp('sandbox-flutter-dart')
# Attempt to update the manifest with a new flutter revision.
with api.step.nest('flutter/flutter'):
flutter_remote = api.auto_roller.update_manifest_project(
manifest=flutter_manifest_path,
project_name=FLUTTER_NAME,
revision=flutter_revision,
updated_deps=updated_deps,
)
if not updated_deps:
return
# Get the flutter/flutter dependency on flutter/engine.
flutter_path = sandbox_dir.join('flutter')
api.git.checkout(
url=flutter_remote,
path=flutter_path,
ref=flutter_revision,
)
engine_revision = api.file.read_text(
name='read flutter engine version',
source=flutter_path.join('bin', 'internal', 'engine.version'),
test_data='xyz000',
).strip()
# Attempt to update the manifest with a new engine revision.
with api.step.nest('flutter/engine'):
# Get the flutter/engine dependency on Dart & Skia.
engine_path = sandbox_dir.join('engine')
api.git.checkout(
url=ENGINE_REMOTE,
path=engine_path,
ref=engine_revision,
)
# Update flutter prebuilts
for prebuilt in FLUTTER_CIPD_PACKAGES:
api.auto_roller.update_manifest_package(
manifest=prebuilt_manifest_path,
package_name=prebuilt,
version='git_revision:%s' % engine_revision,
updated_deps=updated_deps,
)
with api.step.nest('dart sdk'):
dart_revision = extract_dependency_version_from_deps(
api, engine_path.join('DEPS'), 'dart')
dart_remote = api.auto_roller.update_manifest_project(
manifest=dart_manifest_path,
project_name=DART_SDK_NAME,
revision=dart_revision,
updated_deps=updated_deps,
)
# Update prebuilt for dart/sdk
api.jiri.edit_manifest(
manifest=prebuilt_manifest_path,
packages=[(DART_SDK_PKG_NAME, 'git_revision:' + dart_revision)])
# Get dart/sdk.
dart_path = sandbox_dir.join('dart')
api.git.checkout(
url=dart_remote,
path=dart_path,
ref=dart_revision,
)
# Update the package manifests.
with api.step.nest('dart third-party packages'):
update_pkg_manifest(
api,
dart_path=dart_path,
manifest_path=dart_third_party_pkg_manifest_path,
)
# Update fuchsia's third-party dart packages, not to be confused with
# the previous updating of dart's own third-party packages.
with api.step.nest('third-party dart packages'):
third_party_dart_pkg_revision = update_3p_dart_packages(
api=api,
jiri_manifest=api.path['start_dir'].join('.jiri_manifest'),
local_manifest=os.path.relpath(
str(manifest_path),
str(api.path['start_dir']),
),
)
api.auto_roller.update_manifest_project(
manifest=dart_manifest_path,
project_name=THIRD_PARTY_DART_PKG_NAME,
revision=third_party_dart_pkg_revision,
updated_deps=updated_deps,
)
def fetch_and_upload_debug_symbols(api, project, import_in, remote, project_dir,
packages, debug_symbol_attribute,
debug_symbol_gcs_buckets):
"""
Fetch debug symbol archives, unpack them, and upload debug symbols.
Args:
project (str): Jiri remote manifest project.
import_in (str): Path to the edited manifest relative to $project
containing debug symbol packages.
remote (str): Remote manifest repository.
project_dir (Path): Project root path of $import_in.
packages (seq(str)): The list of CIPD packages updated in $import_in.
debug_symbol_attribute (str): Jiri attribute to match debug symbol packages.
debug_symbol_gcs_buckets (seq(str)): GCS buckets to upload debug symbols to.
"""
with api.context(infra_steps=True):
api.jiri.init(
use_lock_file=True,
attributes=(debug_symbol_attribute,),
)
# Fetch debug symbol packages using locally edited manifest.
api.jiri.import_manifest(
manifest=import_in,
remote=remote,
name=project,
)
api.jiri.fetch_packages(local_manifest=True)
with api.step.nest('build'):
gn_results = api.build.gen(
checkout_root=api.path['start_dir'],
fuchsia_build_dir=api.path['start_dir'].join('out', 'default'),
target='x64',
build_type='debug',
product='products/bringup.gni',
# //bundles:infratools is necessary to build upload_debug_symbols.
packages=["//bundles:infratools"],
)
upload_debug_symbols_target = os.path.relpath(
str(gn_results.tool('upload_debug_symbols')),
str(gn_results.fuchsia_build_dir),
)
api.build.ninja(
checkout_root=api.path['start_dir'],
gn_results=gn_results,
build_zircon=False,
targets=[upload_debug_symbols_target],
)
build_id_dirs = []
for package in packages:
# Find archives for each debug symbol package.
with api.context(cwd=project_dir):
package_def = api.jiri.read_manifest_element(
manifest=import_in,
element_type='package',
element_name=package,
)
# Skip non debug symbol packages.
if debug_symbol_attribute not in package_def.get('attributes', ''):
continue
package_path = api.path['start_dir'].join(package_def['path'])
archives = api.file.glob_paths(
name='find archives for %s' % package,
source=package_path,
pattern='**/*.tar.bz2',
test_data=(package_path.join('symbols.tar.bz2'),),
)
# Unpack archives into .build-id dirs.
for archive in archives:
# Extract API requires a unique, non-existent directory.
archive_basename = os.path.basename(api.path.abspath(archive))
output_dir = api.path['start_dir'].join(package, archive_basename)
api.archive.extract(
step_name='extract %s' % archive,
archive_file=archive,
output=output_dir,
)
build_id_dirs.append(output_dir)
for debug_symbol_gcs_bucket in debug_symbol_gcs_buckets:
api.upload_debug_symbols(
step_name='upload debug symbols',
upload_debug_symbols_path=gn_results.tool('upload_debug_symbols'),
bucket=debug_symbol_gcs_bucket,
build_id_dirs=build_id_dirs,
)
def RunSteps(api, manifest, remote, revision, dart_manifest, flutter_manifest,
dart_third_party_pkg_manifest, prebuilt_manifest, lockfiles,
enforce_locks, debug_symbol_attribute, debug_symbol_gcs_buckets):
with api.context(infra_steps=True):
api.jiri.init(use_lock_file=enforce_locks)
api.jiri.import_manifest(
manifest=manifest,
remote=remote,
name='integration',
)
api.jiri.update(run_hooks=False)
flutter_revision = revision or api.buildbucket.build.input.gitiles_commit.id
manifest_repo = api.path['start_dir'].join('integration')
manifest_path = manifest_repo.join(manifest)
flutter_manifest_path = manifest_repo.join(*flutter_manifest.split('/'))
dart_manifest_path = manifest_repo.join(*dart_manifest.split('/'))
dart_third_party_pkg_manifest_path = manifest_repo.join(
*dart_third_party_pkg_manifest.split('/'))
prebuilt_manifest_path = manifest_repo.join(*prebuilt_manifest.split('/'))
updated_deps = OrderedDict()
update_deps(api, updated_deps, manifest_path, flutter_revision,
flutter_manifest_path, dart_manifest_path,
dart_third_party_pkg_manifest_path, prebuilt_manifest_path)
# Update lockfiles
with api.context(cwd=manifest_repo):
for lock_entry in lockfiles:
fields = lock_entry.split('=')
manifest = fields[0]
lock = fields[1]
api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest])
if updated_deps:
rolled = roll_changes(api, updated_deps)
# If debug symbol packages were rolled, upload debug symbols.
# TODO(fxb/37432): Upload debug symbols with artifactory.
packages_updated = set(DEBUG_SYMBOL_PACKAGES).issubset(updated_deps)
if rolled and packages_updated and debug_symbol_gcs_buckets:
with api.step.nest('fetch and upload debug symbols'):
fetch_and_upload_debug_symbols(
api=api,
project='integration',
import_in=prebuilt_manifest,
remote=remote,
project_dir=manifest_repo,
packages=FLUTTER_CIPD_PACKAGES,
debug_symbol_attribute=debug_symbol_attribute,
debug_symbol_gcs_buckets=debug_symbol_gcs_buckets,
)
def GenTests(api):
project_names = {
'flutter/flutter': FLUTTER_NAME,
'dart sdk': DART_SDK_NAME,
'third-party dart packages': THIRD_PARTY_DART_PKG_NAME,
}
noop_edit = lambda name: api.step_data(
'%s.jiri edit %s' % (name, project_names[name]),
api.json.output({'projects': []}),
)
noop_package_edit = lambda name: api.step_data(
'flutter/engine.jiri edit %s' % name,
api.json.output({'packages': []}),
)
def log_data(name):
return api.gitiles.log('%s.log %s' % (name, project_names[name]), 'A')
flutter_check_data = api.jiri.read_manifest_element(
api=api,
manifest='topaz/flutter',
element_name=FLUTTER_NAME,
element_type='project',
test_output={
'remote': 'https://fuchsia.googlesource.com/third_party/flutter',
},
nesting='flutter/flutter')
flutter_log_data = log_data('flutter/flutter')
dart_sdk_check_data = api.jiri.read_manifest_element(
api=api,
manifest='topaz/dart',
element_name=DART_SDK_NAME,
element_type='project',
test_output={
'remote': 'https://fuchsia.googlesource.com/third_party/dart',
},
nesting='dart sdk')
dart_sdk_log_data = log_data('dart sdk')
third_party_dart_pkg_check_data = api.jiri.read_manifest_element(
api=api,
manifest='topaz/dart',
element_name=THIRD_PARTY_DART_PKG_NAME,
element_type='project',
test_output={
'remote': 'https://fuchsia.googlesource.com/third_party/dart-pkg',
},
nesting='third-party dart packages')
third_party_dart_pkg_log_data = log_data('third-party dart packages')
yield (api.test('noop roll') + api.properties(revision='abc123') +
flutter_check_data + noop_edit('flutter/flutter'))
yield (api.test('noop roll with lockfile') + api.properties(
revision='abc123', lockfiles=['manifest/topaz=jiri.lock']) +
flutter_check_data + noop_edit('flutter/flutter'))
yield (api.test('flutter/flutter only') + api.properties(
revision='abc123', debug_symbol_gcs_buckets=('foo-bucket', 'bar-bucket'))
+ flutter_check_data + flutter_log_data + dart_sdk_check_data +
third_party_dart_pkg_check_data + noop_edit('dart sdk') +
noop_edit('third-party dart packages') +
noop_package_edit('flutter/fuchsia-debug-symbols-x64') +
api.auto_roller.success_step_data())
yield (api.test('cannot find dart version') +
api.properties(revision='abc123') + flutter_check_data +
flutter_log_data + api.step_data('dart sdk.read DEPS file for dart',
api.raw_io.output_text('stuff')))
yield (api.test('flutter/flutter and dart') +
api.properties(revision='abc123') + flutter_check_data +
flutter_log_data + dart_sdk_check_data + dart_sdk_log_data +
third_party_dart_pkg_check_data + third_party_dart_pkg_log_data +
api.auto_roller.success_step_data())
yield (api.test('no revision') + flutter_check_data + flutter_log_data +
dart_sdk_check_data + dart_sdk_log_data +
third_party_dart_pkg_check_data + third_party_dart_pkg_log_data +
api.auto_roller.success_step_data() +
api.buildbucket.ci_build(revision='321abc'))
yield (api.test('with debug symbols') + api.properties(
revision='abc123', debug_symbol_gcs_buckets=('foo-bucket', 'bar-bucket'))
+ flutter_check_data + flutter_log_data + dart_sdk_check_data +
third_party_dart_pkg_check_data + noop_edit('dart sdk') +
noop_edit('third-party dart packages') +
api.auto_roller.success_step_data() + api.jiri.read_manifest_element(
api,
'fuchsia/prebuilts',
'package',
'flutter/fuchsia',
test_output={
'path': 'prebuilt/flutter',
},
nesting='fetch and upload debug symbols') +
api.jiri.read_manifest_element(
api,
'fuchsia/prebuilts',
'package',
'flutter/fuchsia-debug-symbols-arm64',
test_output={
'path': 'prebuilt/build_ids/arm64/flutter',
'attributes': 'debug-symbols,debug-symbols-arm64',
},
nesting='fetch and upload debug symbols') +
api.jiri.read_manifest_element(
api,
'fuchsia/prebuilts',
'package',
'flutter/fuchsia-debug-symbols-x64',
test_output={
'path': 'prebuilt/build_ids/x64/flutter',
'attributes': 'debug-symbols,debug-symbols-x64',
},
nesting='fetch and upload debug symbols') +
api.jiri.read_manifest_element(
api,
'fuchsia/prebuilts',
'package',
'flutter/sky_engine',
test_output={
'path': 'prebuilt/sky_engine',
},
nesting='fetch and upload debug symbols'))