recipes/crashpad_roller.py - infra/recipes - Git at Google

 # Copyright 2019 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Recipe for automatically updating Crasphad and its sub-dependencies - mini_chromium & lss."""

 from collections import OrderedDict

 import re

 from recipe_engine.config import List
 from recipe_engine.recipe_api import Property

 DEPS = [
     'fuchsia/auto_roller',
     'fuchsia/buildbucket_util',
     'fuchsia/git',
     'fuchsia/gitiles',
     'fuchsia/jiri',
     'recipe_engine/buildbucket',
     'recipe_engine/context',
     'recipe_engine/file',
     'recipe_engine/json',
     'recipe_engine/path',
     'recipe_engine/properties',
     'recipe_engine/python',
     'recipe_engine/raw_io',
     'recipe_engine/step',
 ]

 PROPERTIES = {
     'project':
         Property(kind=str, help='Jiri remote manifest project', default=None),
     'manifest':
         Property(kind=str, help='Jiri manifest to use'),
     'remote':
         Property(kind=str, help='Remote manifest repository'),
     'import_in':
         Property(
             kind=str,
             help='Path to the manifest to edit relative to $project',
             default='fuchsia/third_party/flower'),
     'revision':
         Property(kind=str, help='Crashpad revision to roll to', default=''),
     'lockfiles':
         Property(
             kind=List(str),
             default=[],
             help='The list of lockfiles to update in "${manifest}=${lockfile}" format'
         ),
 }

 CRASHPAD_PROJECT_NAME = 'third_party/crashpad'
 MINI_CHROMIUM_PROJECT_NAME = 'third_party/mini_chromium'
 LSS_PROJECT_NAME = 'third_party/lss'
 CRASHPAD_REMOTE = 'https://chromium.googlesource.com/crashpad/crashpad'

 COMMIT_MESSAGE = """\
 [roll] Update {deps}

 {logs}

 Cq-Cl-Tag: roller-builder:{builder}
 Cq-Cl-Tag: roller-bid:{build_id}"""

 TEST_DEPS_DATA_TEMPLATE = """\
   '%s':
       Var('chromium_git') + '/linux-syscall-support.git@' +
       '8048ece6c16c91acfe0d36d1d3cc0890ab6e945c',
 """


 def extract_revision_from_deps(api, deps_path, dep_name):
   """Extracts the dependency version from Crashpad's DEPS file.

   Args:
     api (RecipeApi): Recipe API object.
     deps_path (Path): A path to a DEPS file.
     dep_name (str): The name of a dependency to extract from the DEPS file.

   Returns:
     A String containing the revision.
   """
   dep_id = 'crashpad/third_party/%s/%s' % (dep_name, dep_name)
   contents = api.file.read_text(
       name='read DEPS file for %s' % dep_name,
       source=deps_path,
       test_data=TEST_DEPS_DATA_TEMPLATE % dep_id,
   )
   m = re.search(r'\'%s\':[\s\S]*?\'(?P<revision>[0-9a-f]{40})\',' % dep_id,
                 contents)
   if m:
     return m.group('revision')
   raise api.step.InfraFailure('failed to find %s in DEPS' % dep_id)


 def update_crashpad_deps(api, crashpad_revision, manifest):
   """Update manifest for all Crashpad dependencies.

   Args:
     api (RecipeApi): Recipe API object.
     crashpad_revision (str): SHA-1 hash representing the Crashpad revision
       to roll dependencies to.
     manifest (str): Path to the manifest to edit relative to project.

   Returns:
     A dictionary mapping of updated dependencies to their corresponding
     update logs.
   """
   updated_deps = OrderedDict()
   with api.step.nest('crashpad'):
     api.auto_roller.update_manifest_project(
         manifest=manifest,
         project_name=CRASHPAD_PROJECT_NAME,
         revision=crashpad_revision,
         updated_deps=updated_deps)

   # Only check for sub-dependency changes if Crashpad's revision has been updated
   if updated_deps:
     tmp_dir = api.path.mkdtemp('git_checkout')
     crashpad_dir = tmp_dir.join('crashpad')
     crashpad_deps_path = crashpad_dir.join('DEPS')
     # Checkout the Crashpad revision that we are rolling to in order to view
     # its sub-depedencies which are defined in the Crashpad repo's DEPS file
     api.git.checkout(
         url=CRASHPAD_REMOTE,
         path=crashpad_dir,
         ref=crashpad_revision,
     )
     mini_chromium_revision = extract_revision_from_deps(api, crashpad_deps_path,
                                                         'mini_chromium')
     lss_revision = extract_revision_from_deps(api, crashpad_deps_path, 'lss')

     with api.step.nest('mini_chromium'):
       api.auto_roller.update_manifest_project(
           manifest=manifest,
           project_name=MINI_CHROMIUM_PROJECT_NAME,
           revision=mini_chromium_revision,
           updated_deps=updated_deps)

     with api.step.nest('lss'):
       api.auto_roller.update_manifest_project(
           manifest=manifest,
           project_name=LSS_PROJECT_NAME,
           revision=lss_revision,
           updated_deps=updated_deps)
   return updated_deps


 def RunSteps(api, manifest, remote, project, revision, import_in, lockfiles):
   with api.context(infra_steps=True):
     api.jiri.init(use_lock_file=True)
     api.jiri.import_manifest(manifest, remote, project)
     api.jiri.update(run_hooks=False)

   manifest_repo = api.path['start_dir'].join(*project.split('/'))
   manifest_path = manifest_repo.join(*import_in.split('/'))

   if not revision:
     revision = api.gitiles.refs(CRASHPAD_REMOTE).get('refs/heads/master', None)

   updated_deps = update_crashpad_deps(api, revision, manifest_path)

   if not updated_deps:
     api.step('manifest up-to-date; nothing to roll', None)
     return

   # Update lockfiles
   with api.context(cwd=manifest_repo):
     for lock_entry in lockfiles:
       fields = lock_entry.split('=')
       manifest = fields[0]
       lock = fields[1]
       api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest])

   # Land the changes
   commit_message = COMMIT_MESSAGE.format(
       deps=', '.join(updated_deps.keys()),
       logs='\n'.join(updated_deps.itervalues()),
       builder=api.buildbucket.builder_name,
       build_id=api.buildbucket_util.id,
   )
   api.auto_roller.attempt_roll(
       gerrit_project=project,
       repo_dir=manifest_repo,
       commit_message=commit_message,
   )


 def GenTests(api):
   project_names = {
       'crashpad': CRASHPAD_PROJECT_NAME,
       'mini_chromium': MINI_CHROMIUM_PROJECT_NAME,
       'lss': LSS_PROJECT_NAME,
   }

   noop_edit = lambda name: api.step_data(
       '%s.jiri edit %s' % (name, project_names[name]),
       api.json.output({'projects': []}),
   )

   log_data = lambda name: api.gitiles.log(
       '%s.log %s' % (name, project_names[name]), 'A')

   crashpad_check_data = api.jiri.read_manifest_element(
       api,
       manifest='fuchsia/third_party/flower',
       element_name=project_names['crashpad'],
       element_type='project',
       test_output={
           'remote': 'https://chromium.googlesource.com/crashpad/crashpad'
       },
       nesting='crashpad')
   crashpad_log_data = log_data('crashpad')

   mini_chromium_check_data = api.jiri.read_manifest_element(
       api,
       manifest='fuchsia/third_party/flower',
       element_name=project_names['mini_chromium'],
       element_type='project',
       test_output={
           'remote': 'https://chromium.googlesource.com/crashpad/crashpad'
       },
       nesting='mini_chromium')
   mini_chromium_log_data = log_data('mini_chromium')

   lss_check_data = api.jiri.read_manifest_element(
       api,
       manifest='fuchsia/third_party/flower',
       element_name=project_names['lss'],
       element_type='project',
       test_output={
           'remote': 'https://chromium.googlesource.com/crashpad/crashpad'
       },
       nesting='lss')
   lss_log_data = log_data('lss')

   # yapf: disable
   yield (api.test('noop roll') +
          api.properties(revision='abc123',
           project='integration',
           manifest='fuchsia/minimal',
           remote='sso://integration')+
          crashpad_check_data+
          noop_edit('crashpad'))

   yield (api.test('DEPS parsing error') +
          api.properties(revision='abc123',
           project='integration',
           manifest='fuchsia/minimal',
           remote='sso://integration')+
          crashpad_check_data+
          crashpad_log_data+
          api.step_data('read DEPS file for mini_chromium', api.raw_io.output_text('invalid content')))

   yield (api.test('missing_revision') +
          api.properties(project='integration',
           manifest='fuchsia/minimal',
           remote='sso://integration')+
          api.gitiles.refs('refs', (
           'refs/heads/master', 'fc4dc762688d2263b254208f444f5c0a4b91bc07')) +
          crashpad_check_data+
          crashpad_log_data +
          mini_chromium_check_data+
          noop_edit('mini_chromium') +
          lss_check_data+
          noop_edit('lss')+
          api.auto_roller.success_step_data())

   yield (api.test('crashpad roll with lockfile') +
          api.properties(revision='abc123',
           project='integration',
           manifest='fuchsia/minimal',
           remote='sso://integration',
           lockfiles=['manifest/topaz=jiri.lock'])+
          crashpad_check_data+
          crashpad_log_data +
          mini_chromium_check_data+
          noop_edit('mini_chromium') +
          lss_check_data+
          noop_edit('lss')+
          api.auto_roller.success_step_data())

   yield (api.test('crashpad') +
          api.properties(revision='abc123',
           project='integration',
           manifest='fuchsia/minimal',
           remote='sso://integration')+
          crashpad_check_data+
          crashpad_log_data +
          mini_chromium_check_data+
          noop_edit('mini_chromium') +
          lss_check_data+
          noop_edit('lss')+
          api.auto_roller.success_step_data())

   yield (api.test('crashpad and mini_chromium') +
          api.properties(revision='abc123',
           project='integration',
           manifest='fuchsia/minimal',
           remote='sso://integration')+
          crashpad_check_data+
          crashpad_log_data +
          mini_chromium_check_data+
          mini_chromium_log_data+
          lss_check_data+
          noop_edit('lss')+
          api.auto_roller.success_step_data())


   yield (api.test('crashpad, mini_chromium and lss') +
          api.properties(revision='abc123',
           project='integration',
           manifest='fuchsia/minimal',
           remote='sso://integration')+
          crashpad_check_data+
          crashpad_log_data +
          mini_chromium_check_data+
          mini_chromium_log_data +
          lss_check_data+
          lss_log_data+
          api.auto_roller.success_step_data())

   # yapf: enable
	# Copyright 2019 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Recipe for automatically updating Crasphad and its sub-dependencies - mini_chromium & lss."""

	from collections import OrderedDict

	import re

	from recipe_engine.config import List
	from recipe_engine.recipe_api import Property

	DEPS = [
	'fuchsia/auto_roller',
	'fuchsia/buildbucket_util',
	'fuchsia/git',
	'fuchsia/gitiles',
	'fuchsia/jiri',
	'recipe_engine/buildbucket',
	'recipe_engine/context',
	'recipe_engine/file',
	'recipe_engine/json',
	'recipe_engine/path',
	'recipe_engine/properties',
	'recipe_engine/python',
	'recipe_engine/raw_io',
	'recipe_engine/step',
	]

	PROPERTIES = {
	'project':
	Property(kind=str, help='Jiri remote manifest project', default=None),
	'manifest':
	Property(kind=str, help='Jiri manifest to use'),
	'remote':
	Property(kind=str, help='Remote manifest repository'),
	'import_in':
	Property(
	kind=str,
	help='Path to the manifest to edit relative to $project',
	default='fuchsia/third_party/flower'),
	'revision':
	Property(kind=str, help='Crashpad revision to roll to', default=''),
	'lockfiles':
	Property(
	kind=List(str),
	default=[],
	help='The list of lockfiles to update in "${manifest}=${lockfile}" format'
	),
	}

	CRASHPAD_PROJECT_NAME = 'third_party/crashpad'
	MINI_CHROMIUM_PROJECT_NAME = 'third_party/mini_chromium'
	LSS_PROJECT_NAME = 'third_party/lss'
	CRASHPAD_REMOTE = 'https://chromium.googlesource.com/crashpad/crashpad'

	COMMIT_MESSAGE = """\
	[roll] Update {deps}

	{logs}

	Cq-Cl-Tag: roller-builder:{builder}
	Cq-Cl-Tag: roller-bid:{build_id}"""

	TEST_DEPS_DATA_TEMPLATE = """\
	'%s':
	Var('chromium_git') + '/linux-syscall-support.git@' +
	'8048ece6c16c91acfe0d36d1d3cc0890ab6e945c',
	"""


	def extract_revision_from_deps(api, deps_path, dep_name):
	"""Extracts the dependency version from Crashpad's DEPS file.

	Args:
	api (RecipeApi): Recipe API object.
	deps_path (Path): A path to a DEPS file.
	dep_name (str): The name of a dependency to extract from the DEPS file.

	Returns:
	A String containing the revision.
	"""
	dep_id = 'crashpad/third_party/%s/%s' % (dep_name, dep_name)
	contents = api.file.read_text(
	name='read DEPS file for %s' % dep_name,
	source=deps_path,
	test_data=TEST_DEPS_DATA_TEMPLATE % dep_id,
	)
	m = re.search(r'\'%s\':[\s\S]*?\'(?P<revision>[0-9a-f]{40})\',' % dep_id,
	contents)
	if m:
	return m.group('revision')
	raise api.step.InfraFailure('failed to find %s in DEPS' % dep_id)


	def update_crashpad_deps(api, crashpad_revision, manifest):
	"""Update manifest for all Crashpad dependencies.

	Args:
	api (RecipeApi): Recipe API object.
	crashpad_revision (str): SHA-1 hash representing the Crashpad revision
	to roll dependencies to.
	manifest (str): Path to the manifest to edit relative to project.

	Returns:
	A dictionary mapping of updated dependencies to their corresponding
	update logs.
	"""
	updated_deps = OrderedDict()
	with api.step.nest('crashpad'):
	api.auto_roller.update_manifest_project(
	manifest=manifest,
	project_name=CRASHPAD_PROJECT_NAME,
	revision=crashpad_revision,
	updated_deps=updated_deps)

	# Only check for sub-dependency changes if Crashpad's revision has been updated
	if updated_deps:
	tmp_dir = api.path.mkdtemp('git_checkout')
	crashpad_dir = tmp_dir.join('crashpad')
	crashpad_deps_path = crashpad_dir.join('DEPS')
	# Checkout the Crashpad revision that we are rolling to in order to view
	# its sub-depedencies which are defined in the Crashpad repo's DEPS file
	api.git.checkout(
	url=CRASHPAD_REMOTE,
	path=crashpad_dir,
	ref=crashpad_revision,
	)
	mini_chromium_revision = extract_revision_from_deps(api, crashpad_deps_path,
	'mini_chromium')
	lss_revision = extract_revision_from_deps(api, crashpad_deps_path, 'lss')

	with api.step.nest('mini_chromium'):
	api.auto_roller.update_manifest_project(
	manifest=manifest,
	project_name=MINI_CHROMIUM_PROJECT_NAME,
	revision=mini_chromium_revision,
	updated_deps=updated_deps)

	with api.step.nest('lss'):
	api.auto_roller.update_manifest_project(
	manifest=manifest,
	project_name=LSS_PROJECT_NAME,
	revision=lss_revision,
	updated_deps=updated_deps)
	return updated_deps


	def RunSteps(api, manifest, remote, project, revision, import_in, lockfiles):
	with api.context(infra_steps=True):
	api.jiri.init(use_lock_file=True)
	api.jiri.import_manifest(manifest, remote, project)
	api.jiri.update(run_hooks=False)

	manifest_repo = api.path['start_dir'].join(*project.split('/'))
	manifest_path = manifest_repo.join(*import_in.split('/'))

	if not revision:
	revision = api.gitiles.refs(CRASHPAD_REMOTE).get('refs/heads/master', None)

	updated_deps = update_crashpad_deps(api, revision, manifest_path)

	if not updated_deps:
	api.step('manifest up-to-date; nothing to roll', None)
	return

	# Update lockfiles
	with api.context(cwd=manifest_repo):
	for lock_entry in lockfiles:
	fields = lock_entry.split('=')
	manifest = fields[0]
	lock = fields[1]
	api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest])

	# Land the changes
	commit_message = COMMIT_MESSAGE.format(
	deps=', '.join(updated_deps.keys()),
	logs='\n'.join(updated_deps.itervalues()),
	builder=api.buildbucket.builder_name,
	build_id=api.buildbucket_util.id,
	)
	api.auto_roller.attempt_roll(
	gerrit_project=project,
	repo_dir=manifest_repo,
	commit_message=commit_message,
	)


	def GenTests(api):
	project_names = {
	'crashpad': CRASHPAD_PROJECT_NAME,
	'mini_chromium': MINI_CHROMIUM_PROJECT_NAME,
	'lss': LSS_PROJECT_NAME,
	}

	noop_edit = lambda name: api.step_data(
	'%s.jiri edit %s' % (name, project_names[name]),
	api.json.output({'projects': []}),
	)

	log_data = lambda name: api.gitiles.log(
	'%s.log %s' % (name, project_names[name]), 'A')

	crashpad_check_data = api.jiri.read_manifest_element(
	api,
	manifest='fuchsia/third_party/flower',
	element_name=project_names['crashpad'],
	element_type='project',
	test_output={
	'remote': 'https://chromium.googlesource.com/crashpad/crashpad'
	},
	nesting='crashpad')
	crashpad_log_data = log_data('crashpad')

	mini_chromium_check_data = api.jiri.read_manifest_element(
	api,
	manifest='fuchsia/third_party/flower',
	element_name=project_names['mini_chromium'],
	element_type='project',
	test_output={
	'remote': 'https://chromium.googlesource.com/crashpad/crashpad'
	},
	nesting='mini_chromium')
	mini_chromium_log_data = log_data('mini_chromium')

	lss_check_data = api.jiri.read_manifest_element(
	api,
	manifest='fuchsia/third_party/flower',
	element_name=project_names['lss'],
	element_type='project',
	test_output={
	'remote': 'https://chromium.googlesource.com/crashpad/crashpad'
	},
	nesting='lss')
	lss_log_data = log_data('lss')

	# yapf: disable
	yield (api.test('noop roll') +
	api.properties(revision='abc123',
	project='integration',
	manifest='fuchsia/minimal',
	remote='sso://integration')+
	crashpad_check_data+
	noop_edit('crashpad'))

	yield (api.test('DEPS parsing error') +
	api.properties(revision='abc123',
	project='integration',
	manifest='fuchsia/minimal',
	remote='sso://integration')+
	crashpad_check_data+
	crashpad_log_data+
	api.step_data('read DEPS file for mini_chromium', api.raw_io.output_text('invalid content')))

	yield (api.test('missing_revision') +
	api.properties(project='integration',
	manifest='fuchsia/minimal',
	remote='sso://integration')+
	api.gitiles.refs('refs', (
	'refs/heads/master', 'fc4dc762688d2263b254208f444f5c0a4b91bc07')) +
	crashpad_check_data+
	crashpad_log_data +
	mini_chromium_check_data+
	noop_edit('mini_chromium') +
	lss_check_data+
	noop_edit('lss')+
	api.auto_roller.success_step_data())

	yield (api.test('crashpad roll with lockfile') +
	api.properties(revision='abc123',
	project='integration',
	manifest='fuchsia/minimal',
	remote='sso://integration',
	lockfiles=['manifest/topaz=jiri.lock'])+
	crashpad_check_data+
	crashpad_log_data +
	mini_chromium_check_data+
	noop_edit('mini_chromium') +
	lss_check_data+
	noop_edit('lss')+
	api.auto_roller.success_step_data())

	yield (api.test('crashpad') +
	api.properties(revision='abc123',
	project='integration',
	manifest='fuchsia/minimal',
	remote='sso://integration')+
	crashpad_check_data+
	crashpad_log_data +
	mini_chromium_check_data+
	noop_edit('mini_chromium') +
	lss_check_data+
	noop_edit('lss')+
	api.auto_roller.success_step_data())

	yield (api.test('crashpad and mini_chromium') +
	api.properties(revision='abc123',
	project='integration',
	manifest='fuchsia/minimal',
	remote='sso://integration')+
	crashpad_check_data+
	crashpad_log_data +
	mini_chromium_check_data+
	mini_chromium_log_data+
	lss_check_data+
	noop_edit('lss')+
	api.auto_roller.success_step_data())


	yield (api.test('crashpad, mini_chromium and lss') +
	api.properties(revision='abc123',
	project='integration',
	manifest='fuchsia/minimal',
	remote='sso://integration')+
	crashpad_check_data+
	crashpad_log_data +
	mini_chromium_check_data+
	mini_chromium_log_data +
	lss_check_data+
	lss_log_data+
	api.auto_roller.success_step_data())

	# yapf: enable