| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for rolling Fuchsia layers into upper layers.""" |
| |
| from recipe_engine.config import Enum, Single |
| from recipe_engine.recipe_api import Property, StepFailure |
| |
| from urlparse import urlparse |
| |
| # ROLL_TYPES lists the types of rolls we can perform on the target manifest. |
| # * 'import': An <import> tag will be updated. |
| # * 'project': A <project> tag will be updated. |
| ROLL_TYPES = ['import', 'project'] |
| |
| DEPS = [ |
| 'infra/auto_roller', |
| 'infra/gitiles', |
| 'infra/jiri', |
| 'recipe_engine/context', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| ] |
| |
| PROPERTIES = { |
| 'project': |
| Property(kind=str, help='Jiri remote manifest project', default=None), |
| 'manifest': |
| Property(kind=str, help='Jiri manifest to use'), |
| 'remote': |
| Property(kind=str, help='Remote manifest repository'), |
| 'roll_type': |
| Property( |
| kind=Enum(*ROLL_TYPES), |
| help='The type of roll to perform', |
| default='import'), |
| 'import_in': |
| Property( |
| kind=str, help='Path to the manifest to edit relative to $project'), |
| 'import_from': |
| Property( |
| kind=str, |
| help='Name of the <project> or <import> to edit in $import_in'), |
| 'revision': |
| Property(kind=str, help='Revision', default=None), |
| 'dry_run': |
| Property( |
| kind=bool, |
| default=False, |
| help= |
| 'Whether to dry-run the auto-roller (CQ+1 and abandon the change)'), |
| } |
| |
| COMMIT_MESSAGE = """[roll] Roll {project} {old}..{new} ({count} commits) |
| |
| {commits} |
| |
| Test: CQ |
| """ |
| |
| def SsoToHttps(remote): |
| """Transform a url of SSO scheme to its associated https version """ |
| if not remote or 'sso://' not in remote: |
| return remote |
| url = urlparse(remote) |
| # Note that url.path contains a leading '/'. |
| return 'https://%s.googlesource.com%s' % (url.netloc, url.path) |
| |
| # This recipe has two 'modes' of operation: production and dry-run. Which mode |
| # of execution should be used is dictated by the 'dry_run' property. |
| # |
| # The purpose of dry-run mode is to test the auto-roller end-to-end. This is |
| # useful because now we can have an auto-roller in staging, and we can block |
| # updates behind 'dry_run' as a sort of feature gate. It is passed to |
| # api.auto_roller.attempt_roll() which handles committing changes. |
| def RunSteps(api, project, manifest, remote, roll_type, import_in, import_from, |
| revision, dry_run): |
| api.jiri.ensure_jiri() |
| api.gitiles.ensure_gitiles() |
| |
| with api.context(infra_steps=True): |
| api.jiri.init() |
| api.jiri.import_manifest(manifest, remote, name=project) |
| api.jiri.update(run_hooks=False) |
| |
| project_dir = api.path['start_dir'].join(*project.split('/')) |
| with api.context(cwd=project_dir): |
| # Read the remote URL of the repo we're rolling from. |
| roll_from_repo = SsoToHttps(api.jiri.read_manifest_element( |
| manifest=import_in, |
| element_type=roll_type, |
| element_name=import_from, |
| ).get('remote')) |
| |
| if not revision: |
| revision = api.gitiles.refs(roll_from_repo).get('refs/heads/master', None) |
| |
| # Determine whether to update manifest imports or projects. |
| if roll_type == 'import': |
| updated_section = 'imports' |
| imports = [(import_from, revision)] |
| projects = None |
| elif roll_type == 'project': |
| updated_section = 'projects' |
| imports = None |
| projects = [(import_from, revision)] |
| |
| changes = api.jiri.edit_manifest( |
| import_in, projects=projects, imports=imports) |
| |
| if len(changes[updated_section]) == 0: |
| api.step.active_result.presentation.step_text = 'manifest up-to-date, nothing to roll' |
| return |
| old_rev = changes[updated_section][0]['old_revision'] |
| new_rev = changes[updated_section][0]['new_revision'] |
| |
| # Fail if the remote URL is missing |
| if not roll_from_repo: |
| raise api.step.StepFailure('%s missing remote= attribute' % import_from) |
| |
| # After rolling the commit, re-update and emit a source manifest. |
| # Emitting a source manifest is necessary for luci-notify to be able to |
| # pick up on a checkout diff, allowing it to notify the blamelist across |
| # the whole checkout. Note that we need to say "local_manifest=True" |
| # because otherwise jiri won't use the pin we just updated in the jiri |
| # update, so the emitted source manifest won't contain the updated pin. |
| # |
| # Note that local_manifest=True only works in this way if the pin we're |
| # updating is for the same manifest that we originally pulled from (i.e. |
| # the manifest property) since otherwise the manifest update will just |
| # get ignored and jiri will error (it will never overwrite a dirty |
| # repository). |
| api.jiri.update(run_hooks=False, local_manifest=True) |
| api.jiri.emit_source_manifest() |
| |
| # Get the commit history and generate a commit message. |
| log = api.gitiles.log( |
| roll_from_repo, '%s..%s' % (old_rev, new_rev), step_name='log') |
| message = COMMIT_MESSAGE.format( |
| project=import_from, |
| old=old_rev[:7], |
| new=new_rev[:7], |
| count=len(log), |
| commits='\n'.join([ |
| '{commit} {subject}'.format( |
| commit=commit['id'][:7], |
| subject=commit['message'].splitlines()[0], |
| ) for commit in log |
| ]), |
| ) |
| |
| # Land the changes. |
| api.auto_roller.attempt_roll( |
| gerrit_project=project, |
| repo_dir=project_dir, |
| commit_message=message, |
| dry_run=dry_run, |
| ) |
| |
| |
| # yapf: disable |
| def GenTests(api): |
| # Mock step data intended to be substituted as the result of the first check |
| # during polling. It indicates a success, and should end polling. |
| success_step_data = api.step_data('check if done (0)', api.auto_roller.success()) |
| |
| # Test when the incoming revision is missing. |
| yield (api.test('missing_revision') + |
| # Set test input properties. |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| remote='https://fuchsia.googlesource.com/garnet', |
| import_in='manifest/third_party', |
| import_from='zircon') + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'}) + |
| api.gitiles.refs('refs', ( |
| 'refs/heads/master', 'fc4dc762688d2263b254208f444f5c0a4b91bc07')) + |
| api.gitiles.log('log', 'A') + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'})) |
| |
| # Test when the project to roll from is missing a 'remote' manifest attribute. |
| yield (api.test('missing_manifest_project_remote') + |
| # Set test input properties. |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| remote='https://fuchsia.googlesource.com/garnet', |
| import_in='manifest/third_party', |
| roll_type='project', |
| import_from='cobalt', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| # Generate step data. Mock a call to JiriApi.read_manifest_element. |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='cobalt', |
| element_type='project', |
| test_output={})) |
| |
| # Test when the import to roll from is missing a 'remote' manifest attribute. |
| yield (api.test('missing_manifest_import_remote') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| remote='https://fuchsia.googlesource.com/garnet', |
| import_in='manifest/garnet', |
| roll_type='import', |
| import_from='zircon', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={})) |
| |
| # Test rolling a project instead of an import. |
| yield (api.test('cobalt_project') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| remote='https://fuchsia.googlesource.com/garnet', |
| import_in='manifest/third_party', |
| roll_type='project', |
| import_from='cobalt', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A') + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='cobalt', |
| element_type='project', |
| test_output={ |
| 'remote': 'https://cobalt-analytics.googlesource.com/config' |
| })) |
| |
| # Test a successful roll of zircon into garnet. |
| yield (api.test('zircon') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A') + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'})) |
| |
| # Test a successful roll of zircon into garnet. |
| yield (api.test('roll_from_non_https_remote') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='third_party/foo', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A') + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='third_party/foo', |
| element_type='import', |
| test_output={'remote': 'sso://host/foo'})) |
| |
| # Test a no-op roll of zircon into garnet. |
| yield (api.test('zircon-noop') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'}) + |
| api.step_data('jiri edit', api.json.output({'imports': []}))) |
| |
| # Test a successful roll of garnet into peridot. |
| yield (api.test('garnet') + |
| api.properties(project='peridot', |
| manifest='manifest/peridot', |
| import_in='manifest/peridot', |
| import_from='garnet', |
| remote='https://fuchsia.googlesource.com/peridot', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A') + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='garnet', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/garnet'})) |
| |
| # Test a successful roll of peridot into topaz. |
| yield (api.test('peridot') + |
| api.properties(project='topaz', |
| manifest='manifest/topaz', |
| import_in='manifest/topaz', |
| import_from='peridot', |
| remote='https://fuchsia.googlesource.com/topaz', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07') + |
| api.gitiles.log('log', 'A') + success_step_data + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='peridot', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/peridot'})) |
| |
| # Test a dry-run of the auto-roller for rolling zircon into garnet. We |
| # substitute in mock data for the first check that the CQ dry-run completed by |
| # unsetting the CQ label to indicate that the CQ dry-run finished. |
| yield (api.test('zircon_dry_run') + |
| api.properties(project='garnet', |
| manifest='manifest/garnet', |
| import_in='manifest/garnet', |
| import_from='zircon', |
| remote='https://fuchsia.googlesource.com/garnet', |
| revision='fc4dc762688d2263b254208f444f5c0a4b91bc07', |
| dry_run=True) + |
| api.gitiles.log('log', 'A') + |
| api.step_data('check if done (0)', api.auto_roller.dry_run()) + |
| api.jiri.read_manifest_element(api, |
| manifest='manifest/garnet', |
| element_name='zircon', |
| element_type='import', |
| test_output={'remote': 'https://fuchsia.googlesource.com/zircon'})) |