| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for automatically updating Crasphad and its sub-dependencies - mini_chromium & lss.""" |
| |
| from collections import OrderedDict |
| |
| import re |
| |
| from recipe_engine.config import List |
| from recipe_engine.recipe_api import Property |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/git', |
| 'fuchsia/gitiles', |
| 'fuchsia/jiri', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/python', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = { |
| 'project': |
| Property(kind=str, help='Jiri remote manifest project', default=None), |
| 'manifest': |
| Property(kind=str, help='Jiri manifest to use'), |
| 'remote': |
| Property(kind=str, help='Remote manifest repository'), |
| 'import_in': |
| Property( |
| kind=str, |
| help='Path to the manifest to edit relative to $project', |
| default='fuchsia/third_party/flower'), |
| 'revision': |
| Property(kind=str, help='Crashpad revision to roll to', default=''), |
| 'lockfiles': |
| Property( |
| kind=List(str), |
| default=[], |
| help='The list of lockfiles to update in "${manifest}=${lockfile}" format' |
| ), |
| } |
| |
| CRASHPAD_PROJECT_NAME = 'third_party/crashpad' |
| MINI_CHROMIUM_PROJECT_NAME = 'third_party/mini_chromium' |
| LSS_PROJECT_NAME = 'third_party/lss' |
| CRASHPAD_REMOTE = 'https://chromium.googlesource.com/crashpad/crashpad' |
| |
| COMMIT_MESSAGE = """\ |
| [roll] Update {deps} |
| |
| {logs} |
| |
| Cq-Cl-Tag: roller-builder:{builder} |
| Cq-Cl-Tag: roller-bid:{build_id} |
| """ |
| |
| TEST_DEPS_DATA_TEMPLATE = """\ |
| '%s': |
| Var('chromium_git') + '/linux-syscall-support.git@' + |
| '8048ece6c16c91acfe0d36d1d3cc0890ab6e945c', |
| """ |
| |
| |
| def extract_revision_from_deps(api, deps_path, dep_name): |
| """Extracts the dependency version from Crashpad's DEPS file. |
| |
| Args: |
| api (RecipeApi): Recipe API object. |
| deps_path (Path): A path to a DEPS file. |
| dep_name (str): The name of a dependency to extract from the DEPS file. |
| |
| Returns: |
| A String containing the revision. |
| """ |
| dep_id = 'crashpad/third_party/%s/%s' % (dep_name, dep_name) |
| contents = api.file.read_text( |
| name='read DEPS file for %s' % dep_name, |
| source=deps_path, |
| test_data=TEST_DEPS_DATA_TEMPLATE % dep_id, |
| ) |
| m = re.search(r'\'%s\':[\s\S]*?\'(?P<revision>[0-9a-f]{40})\',' % dep_id, |
| contents) |
| if m: |
| return m.group('revision') |
| raise api.step.InfraFailure('failed to find %s in DEPS' % dep_id) |
| |
| |
| def update_crashpad_deps(api, crashpad_revision, manifest): |
| """Update manifest for all Crashpad dependencies. |
| |
| Args: |
| api (RecipeApi): Recipe API object. |
| crashpad_revision (str): SHA-1 hash representing the Crashpad revision |
| to roll dependencies to. |
| manifest (str): Path to the manifest to edit relative to project. |
| |
| Returns: |
| A dictionary mapping of updated dependencies to their corresponding |
| update logs. |
| """ |
| updated_deps = OrderedDict() |
| with api.step.nest('crashpad'): |
| api.auto_roller.update_manifest_project( |
| manifest=manifest, |
| project_name=CRASHPAD_PROJECT_NAME, |
| revision=crashpad_revision, |
| updated_deps=updated_deps) |
| |
| # Only check for sub-dependency changes if Crashpad's revision has been updated |
| if updated_deps: |
| tmp_dir = api.path.mkdtemp('git_checkout') |
| crashpad_dir = tmp_dir.join('crashpad') |
| crashpad_deps_path = crashpad_dir.join('DEPS') |
| # Checkout the Crashpad revision that we are rolling to in order to view |
| # its sub-depedencies which are defined in the Crashpad repo's DEPS file |
| api.git.checkout( |
| url=CRASHPAD_REMOTE, |
| path=crashpad_dir, |
| ref=crashpad_revision, |
| ) |
| mini_chromium_revision = extract_revision_from_deps(api, crashpad_deps_path, |
| 'mini_chromium') |
| lss_revision = extract_revision_from_deps(api, crashpad_deps_path, 'lss') |
| |
| with api.step.nest('mini_chromium'): |
| api.auto_roller.update_manifest_project( |
| manifest=manifest, |
| project_name=MINI_CHROMIUM_PROJECT_NAME, |
| revision=mini_chromium_revision, |
| updated_deps=updated_deps) |
| |
| with api.step.nest('lss'): |
| api.auto_roller.update_manifest_project( |
| manifest=manifest, |
| project_name=LSS_PROJECT_NAME, |
| revision=lss_revision, |
| updated_deps=updated_deps) |
| return updated_deps |
| |
| |
| def RunSteps(api, manifest, remote, project, revision, import_in, lockfiles): |
| with api.context(infra_steps=True): |
| api.jiri.init(use_lock_file=True) |
| api.jiri.import_manifest(manifest, remote, project) |
| api.jiri.update(run_hooks=False) |
| |
| manifest_repo = api.path['start_dir'].join(*project.split('/')) |
| manifest_path = manifest_repo.join(*import_in.split('/')) |
| |
| if not revision: |
| revision = api.gitiles.refs(CRASHPAD_REMOTE).get('refs/heads/master', None) |
| |
| updated_deps = update_crashpad_deps(api, revision, manifest_path) |
| |
| if not updated_deps: |
| api.step('manifest up-to-date; nothing to roll', None) |
| return |
| |
| # Update lockfiles |
| with api.context(cwd=manifest_repo): |
| for lock_entry in lockfiles: |
| fields = lock_entry.split('=') |
| manifest = fields[0] |
| lock = fields[1] |
| api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest]) |
| |
| # Land the changes |
| commit_message = COMMIT_MESSAGE.format( |
| deps=', '.join(updated_deps.keys()), |
| logs='\n'.join(updated_deps.itervalues()), |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| api.auto_roller.attempt_roll( |
| gerrit_project=project, |
| repo_dir=manifest_repo, |
| commit_message=commit_message, |
| ) |
| |
| |
| def GenTests(api): |
| project_names = { |
| 'crashpad': CRASHPAD_PROJECT_NAME, |
| 'mini_chromium': MINI_CHROMIUM_PROJECT_NAME, |
| 'lss': LSS_PROJECT_NAME, |
| } |
| |
| noop_edit = lambda name: api.step_data( |
| '%s.jiri edit %s' % (name, project_names[name]), |
| api.json.output({'projects': []}), |
| ) |
| |
| log_data = lambda name: api.gitiles.log( |
| '%s.log %s' % (name, project_names[name]), 'A') |
| |
| crashpad_check_data = api.jiri.read_manifest_element( |
| api, |
| manifest='fuchsia/third_party/flower', |
| element_name=project_names['crashpad'], |
| element_type='project', |
| test_output={ |
| 'remote': 'https://chromium.googlesource.com/crashpad/crashpad' |
| }, |
| nesting='crashpad') |
| crashpad_log_data = log_data('crashpad') |
| |
| mini_chromium_check_data = api.jiri.read_manifest_element( |
| api, |
| manifest='fuchsia/third_party/flower', |
| element_name=project_names['mini_chromium'], |
| element_type='project', |
| test_output={ |
| 'remote': 'https://chromium.googlesource.com/crashpad/crashpad' |
| }, |
| nesting='mini_chromium') |
| mini_chromium_log_data = log_data('mini_chromium') |
| |
| lss_check_data = api.jiri.read_manifest_element( |
| api, |
| manifest='fuchsia/third_party/flower', |
| element_name=project_names['lss'], |
| element_type='project', |
| test_output={ |
| 'remote': 'https://chromium.googlesource.com/crashpad/crashpad' |
| }, |
| nesting='lss') |
| lss_log_data = log_data('lss') |
| |
| # yapf: disable |
| yield (api.test('noop roll') + |
| api.properties(revision='abc123', |
| project='integration', |
| manifest='fuchsia/minimal', |
| remote='sso://integration')+ |
| crashpad_check_data+ |
| noop_edit('crashpad')) |
| |
| yield (api.test('DEPS parsing error') + |
| api.properties(revision='abc123', |
| project='integration', |
| manifest='fuchsia/minimal', |
| remote='sso://integration')+ |
| crashpad_check_data+ |
| crashpad_log_data+ |
| api.step_data('read DEPS file for mini_chromium', api.raw_io.output_text('invalid content'))) |
| |
| yield (api.test('missing_revision') + |
| api.properties(project='integration', |
| manifest='fuchsia/minimal', |
| remote='sso://integration')+ |
| api.gitiles.refs('refs', ( |
| 'refs/heads/master', 'fc4dc762688d2263b254208f444f5c0a4b91bc07')) + |
| crashpad_check_data+ |
| crashpad_log_data + |
| mini_chromium_check_data+ |
| noop_edit('mini_chromium') + |
| lss_check_data+ |
| noop_edit('lss')+ |
| api.auto_roller.success_step_data()) |
| |
| yield (api.test('crashpad roll with lockfile') + |
| api.properties(revision='abc123', |
| project='integration', |
| manifest='fuchsia/minimal', |
| remote='sso://integration', |
| lockfiles=['manifest/topaz=jiri.lock'])+ |
| crashpad_check_data+ |
| crashpad_log_data + |
| mini_chromium_check_data+ |
| noop_edit('mini_chromium') + |
| lss_check_data+ |
| noop_edit('lss')+ |
| api.auto_roller.success_step_data()) |
| |
| yield (api.test('crashpad') + |
| api.properties(revision='abc123', |
| project='integration', |
| manifest='fuchsia/minimal', |
| remote='sso://integration')+ |
| crashpad_check_data+ |
| crashpad_log_data + |
| mini_chromium_check_data+ |
| noop_edit('mini_chromium') + |
| lss_check_data+ |
| noop_edit('lss')+ |
| api.auto_roller.success_step_data()) |
| |
| yield (api.test('crashpad and mini_chromium') + |
| api.properties(revision='abc123', |
| project='integration', |
| manifest='fuchsia/minimal', |
| remote='sso://integration')+ |
| crashpad_check_data+ |
| crashpad_log_data + |
| mini_chromium_check_data+ |
| mini_chromium_log_data+ |
| lss_check_data+ |
| noop_edit('lss')+ |
| api.auto_roller.success_step_data()) |
| |
| |
| yield (api.test('crashpad, mini_chromium and lss') + |
| api.properties(revision='abc123', |
| project='integration', |
| manifest='fuchsia/minimal', |
| remote='sso://integration')+ |
| crashpad_check_data+ |
| crashpad_log_data + |
| mini_chromium_check_data+ |
| mini_chromium_log_data + |
| lss_check_data+ |
| lss_log_data+ |
| api.auto_roller.success_step_data()) |
| |
| # yapf: enable |