| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling Fuchsia layers into upper layers.""" |
| |
| from recipe_engine.config import Enum, List |
| from recipe_engine.recipe_api import Property |
| |
| import re |
| from urlparse import urlparse |
| |
| # ROLL_TYPES lists the types of rolls we can perform on the target manifest. |
| # * 'import': An <import> tag will be updated. |
| # * 'project': A <project> tag will be updated. |
| ROLL_TYPES = ['import', 'project'] |
| |
| DEPS = [ |
| 'fuchsia/auto_roller', |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/gerrit', |
| 'fuchsia/gitiles', |
| 'fuchsia/jiri', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = { |
| 'project': |
| Property(kind=str, help='Jiri remote manifest project', default=None), |
| 'manifest': |
| Property(kind=str, help='Jiri manifest to use'), |
| 'remote': |
| Property(kind=str, help='Remote manifest repository'), |
| 'roll_type': |
| Property( |
| kind=Enum(*ROLL_TYPES), |
| help='The type of roll to perform', |
| default='import'), |
| 'import_in': |
| Property( |
| kind=str, help='Path to the manifest to edit relative to $project'), |
| 'import_from': |
| Property( |
| kind=str, |
| help='Name of the <project> or <import> to edit in $import_in'), |
| 'revision': |
| Property(kind=str, help='Revision', default=None), |
| 'lockfiles': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('The list of lockfiles to update in ' |
| '"${manifest}=${lockfile}" format')), |
| # TODO: delete this property after we have full lockfile support in |
| # integration repo |
| 'enforce_locks': |
| Property( |
| kind=bool, |
| default=False, |
| help='Whether to enforce locks from lockfiles'), |
| 'dry_run': |
| Property( |
| kind=bool, |
| default=False, |
| help=('Whether to dry-run the auto-roller (CQ+1 and abandon the ' |
| 'change)')), |
| 'send_comment': |
| Property( |
| kind=bool, |
| default=True, |
| help='Whether to comment on the rolled commits once roll is complete' |
| ), |
| 'owners': |
| Property( |
| kind=List(str), |
| default=(), |
| help=('The owners responsible for watching this roller ' |
| '(example: "username@google.com").')), |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {project} {old}..{new} ({count} commits) |
| |
| {commits} |
| |
| Cq-Cl-Tag: roller-builder:{builder} |
| Cq-Cl-Tag: roller-bid:{build_id} |
| CQ-Do-Not-Cancel-Tryjobs: true""" |
| |
| |
| def sso_to_https(remote): |
| """Transform a url of SSO scheme to its associated https version """ |
| if not remote or 'sso://' not in remote: |
| return remote |
| url = urlparse(remote) |
| # Note that url.path contains a leading '/'. |
| return 'https://%s.googlesource.com%s' % (url.netloc, url.path) |
| |
| |
| # This recipe has two 'modes' of operation: production and dry-run. Which mode |
| # of execution should be used is dictated by the 'dry_run' property. |
| # |
| # The purpose of dry-run mode is to test the auto-roller end-to-end. This is |
| # useful because now we can have an auto-roller in staging, and we can block |
| # updates behind 'dry_run' as a sort of feature gate. It is passed to |
| # api.auto_roller.attempt_roll() which handles committing changes. |
| def RunSteps(api, project, manifest, remote, roll_type, import_in, import_from, |
| revision, lockfiles, enforce_locks, dry_run, send_comment, owners): |
| with api.context(infra_steps=True): |
| if owners: |
| owners_step = api.step('owners', None) |
| owners_step.presentation.step_summary_text = ', '.join(owners) |
| |
| api.jiri.init(use_lock_file=enforce_locks) |
| api.jiri.import_manifest(manifest, remote, name=project) |
| api.jiri.update(run_hooks=False) |
| |
| project_dir = api.path['start_dir'].join(*project.split('/')) |
| with api.context(cwd=project_dir): |
| # Read the remote URL of the repo we're rolling from. |
| origin_manifest_element = api.jiri.read_manifest_element( |
| manifest=import_in, |
| element_type=roll_type, |
| element_name=import_from, |
| ) |
| |
| roll_from_repo = sso_to_https(origin_manifest_element.get('remote')) |
| |
| if not revision: |
| revision = api.gitiles.refs(roll_from_repo).get('refs/heads/master', |
| None) |
| |
| # Determine whether to update manifest imports or projects. |
| if roll_type == 'import': |
| updated_section = 'imports' |
| imports = [(import_from, revision)] |
| projects = None |
| elif roll_type == 'project': |
| updated_section = 'projects' |
| imports = None |
| projects = [(import_from, revision)] |
| |
| changes = api.jiri.edit_manifest( |
| import_in, projects=projects, imports=imports) |
| for lock_entry in lockfiles: |
| fields = lock_entry.split('=') |
| manifest = fields[0] |
| lock = fields[1] |
| api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest]) |
| |
| if not changes[updated_section]: |
| api.step.active_result.presentation.step_text = 'manifest up-to-date, nothing to roll' |
| return |
| old_rev = changes[updated_section][0]['old_revision'] |
| new_rev = changes[updated_section][0]['new_revision'] |
| |
| # Fail if the remote URL is missing |
| if not roll_from_repo: |
| raise api.step.StepFailure('%s missing remote= attribute' % import_from) |
| |
| # After rolling the commit, re-update and emit a source manifest. |
| # Emitting a source manifest is necessary for luci-notify to be able to |
| # pick up on a checkout diff, allowing it to notify the blamelist across |
| # the whole checkout. Note that we need to say "local_manifest=True" |
| # because otherwise jiri won't use the pin we just updated in the jiri |
| # update, so the emitted source manifest won't contain the updated pin. |
| # |
| # Note that local_manifest=True only works in this way if the pin we're |
| # updating is for the same manifest that we originally pulled from (i.e. |
| # the manifest property) since otherwise the manifest update will just |
| # get ignored and jiri will error (it will never overwrite a dirty |
| # repository). |
| api.jiri.update(run_hooks=False, local_manifest=True) |
| api.jiri.emit_source_manifest() |
| |
| # Get the commit history and generate a commit message. |
| log = api.gitiles.log( |
| roll_from_repo, '%s..%s' % (old_rev, new_rev), step_name='log') |
| origin_host = origin_manifest_element.get('gerrithost') |
| current_host = api.gerrit.host |
| if origin_host: |
| # change gerrit host to get change details from correct host |
| api.gerrit.host = origin_host |
| |
| commit_lines = [] |
| |
| with api.step.nest('get gerrit change numbers'): |
| for commit in log: |
| commit_label = commit['id'][:7] |
| |
| if origin_host: |
| change = api.gerrit.change_details( |
| 'change details for %s' % commit_label, |
| commit['id'], |
| test_data=api.json.test_api.output({'_number': 12345}), |
| ok_ret='any') |
| if change: |
| # url.path contains a leading '/'. |
| project_repo = urlparse(roll_from_repo).path |
| commit_label = '{label}:{gerrit_host}/c{project_repo}/+/{change_number}'.format( |
| label=commit_label, |
| gerrit_host=origin_host.rstrip('/'), |
| project_repo=project_repo, |
| change_number=str(change['_number'])) |
| commit_lines.append('{commit} {subject}'.format( |
| commit=commit_label, |
| subject=commit['message'].splitlines()[0], |
| )) |
| |
| # reset host |
| api.gerrit.host = current_host |
| |
| message = COMMIT_MESSAGE.format( |
| project=import_from, |
| old=old_rev[:7], |
| new=new_rev[:7], |
| count=len(log), |
| commits='\n'.join(commit_lines), |
| builder=api.buildbucket.builder_name, |
| build_id=api.buildbucket_util.id, |
| ) |
| |
| # Check for Change-Id in the commit message. If no Change-Id is in the |
| # message ignore and continue; attempt_roll() will just skip commenting |
| # on those revisions. Still passing revision hash into attempt_roll() |
| # because Change-Ids are sometimes copied when cherry-picking and are |
| # less reliable than a revision hash at uniquely identifying a CL. |
| commit_ids = [] |
| for commit in log: |
| match = re.search(r'^Change-Id:\s*(\w+)\s*$', commit['message'], |
| re.MULTILINE) |
| if match: |
| commit_ids.append(commit['id']) |
| |
| # Land the changes. |
| api.auto_roller.attempt_roll( |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| origin_host=origin_host, |
| commit_ids=commit_ids, |
| send_comment=send_comment) |
| |
| |
| # yapf: disable |
| def GenTests(api): |
| # Mock step data intended to be substituted as the result of the first check |
| # during polling. It indicates a success, and should end polling. |
| success_step_data = api.auto_roller.success_step_data() |
| |
| # Test when the incoming revision is missing. |
| yield (api.test('missing_revision') + |
| # Set test input properties. |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| remote='https://fuchsia.googlesource.com/garnet', |
| import_in='manifest/third_party', |
| import_from='zircon', |
| owners=['nobody@google.com', 'noreply@google.com']) + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'}) + |
| api.gitiles.refs('refs', ( |
| 'refs/heads/master', 'fc4dc762688d2263b254208f444f5c0a4b91bc07')) + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'})) |
| |
| # Test when the project to roll from is missing a 'remote' manifest attribute. |
| yield (api.test('missing_manifest_project_remote') + |
| # Set test input properties. |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| remote='https://fuchsia.googlesource.com/garnet', |
| import_in='manifest/third_party', |
| roll_type='project', |
| import_from='cobalt', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| # Generate step data. Mock a call to JiriApi.read_manifest_element. |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='cobalt', |
| element_type='project', |
| test_output={})) |
| |
| # Test when the import to roll from is missing a 'remote' manifest attribute. |
| yield (api.test('missing_manifest_import_remote') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| remote='https://fuchsia.googlesource.com/garnet', |
| import_in='manifest/garnet', |
| roll_type='import', |
| import_from='zircon', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={})) |
| |
| # Test rolling a project instead of an import. |
| yield (api.test('cobalt_project') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| remote='https://fuchsia.googlesource.com/garnet', |
| import_in='manifest/third_party', |
| roll_type='project', |
| import_from='cobalt', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='cobalt', |
| element_type='project', |
| test_output={ |
| 'remote': 'https://cobalt-analytics.googlesource.com/config' |
| })) |
| |
| # Test a successful roll of zircon into garnet. |
| yield (api.test('zircon') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'})) |
| |
| # Test a successful roll of zircon into garnet. |
| yield (api.test('roll_from_non_https_remote') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='third_party/foo', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='third_party/foo', |
| element_type='import', |
| test_output={'remote': 'sso://host/foo'})) |
| |
| # Test a no-op roll of zircon into garnet. |
| yield (api.test('zircon-noop') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'}) + |
| api.step_data('jiri edit', api.json.output({'imports': []}))) |
| |
| # Test a successful roll of garnet into peridot. |
| yield (api.test('garnet') + |
| api.properties(project='peridot', |
| manifest='manifest/peridot', |
| import_in='manifest/peridot', |
| import_from='garnet', |
| remote='https://fuchsia.googlesource.com/peridot', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='garnet', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/garnet'})) |
| |
| # Test a successful roll of peridot into topaz. |
| yield (api.test('peridot') + |
| api.properties(project='topaz', |
| manifest='manifest/topaz', |
| import_in='manifest/topaz', |
| import_from='peridot', |
| remote='https://fuchsia.googlesource.com/topaz', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='peridot', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/peridot'})) |
| |
| # Test a dry-run of the auto-roller for rolling zircon into garnet. We |
| # substitute in mock data for the first check that the CQ dry-run completed by |
| # unsetting the CQ label to indicate that the CQ dry-run finished. |
| yield (api.test('zircon_dry_run') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07', |
| dry_run=True) + |
| api.gitiles.log('log', 'A', add_change_id=True) + |
| api.auto_roller.dry_run_step_data() + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'})) |
| |
| # Test a successful roll of peridot into topaz with lockfile. |
| yield (api.test('peridot_with_lockfile') + |
| api.properties(project='topaz', |
| manifest='manifest/topaz', |
| import_in='manifest/topaz', |
| import_from='peridot', |
| remote='https://fuchsia.googlesource.com/topaz', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07', |
| lockfiles=['manifest/topaz=jiri.lock']) + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='peridot', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/peridot'})) |
| |
| # Test a successful roll without sending a comment once the roll is complete. |
| yield (api.test('zircon_no_comment') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07', |
| send_comment=False) + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'})) |
| |
| # Test a successful roll without sending a comment because the change has |
| # no Change-Id. |
| yield (api.test('zircon_no_changeid') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A') + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'})) |
| |
| # Test a successful roll with links to the rolled commits in the commit |
| # message. |
| yield (api.test('zircon_with_commit_links') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A', add_change_id=True) + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon', |
| 'gerrithost': 'https://origin-host-review.googlesource.com'})) |
