| # Copyright 2016 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from recipe_engine import recipe_api |
| |
| # Flags added to all jiri commands. |
| COMMON_FLAGS = ['-vv', '-time', '-j=50'] |
| |
| # Pinned jiri version. |
| JIRI_VERSION = 'git_revision:59478f6e3bfff1e6794f337c6c38a769bf0c8422' |
| |
| |
| class JiriApi(recipe_api.RecipeApi): |
| """JiriApi provides support for Jiri managed checkouts.""" |
| |
| class RebaseError(recipe_api.StepFailure): |
| # Keep in sync with |
| # https://fuchsia.googlesource.com/jiri/+/master/cmd/jiri/patch.go |
| EXIT_CODE = 24 |
| |
| def __init__(self, *args, **kwargs): |
| super(JiriApi, self).__init__(*args, **kwargs) |
| self._jiri_executable = None |
| |
| def __call__(self, *args, **kwargs): |
| """Return a jiri command step.""" |
| self._ensure() |
| |
| subcommand = args[0] # E.g., 'init' or 'update' |
| flags = COMMON_FLAGS + list(args[1:]) |
| |
| full_cmd = [self._jiri_executable, subcommand] + flags |
| |
| name = kwargs.pop('name', 'jiri ' + subcommand) |
| return self.m.step(name, full_cmd, **kwargs) |
| |
| def _ensure(self): |
| if not self._jiri_executable: |
| with self.m.step.nest('ensure jiri'): |
| with self.m.context(infra_steps=True): |
| pkgs = self.m.cipd.EnsureFile() |
| pkgs.add_package('fuchsia/tools/jiri/${platform}', JIRI_VERSION) |
| cipd_dir = self.m.path['start_dir'].join('cipd', 'jiri') |
| self.m.cipd.ensure(cipd_dir, pkgs) |
| self._jiri_executable = cipd_dir.join('jiri') |
| |
| @property |
| def jiri(self): |
| return self._jiri_executable |
| |
| def init(self, directory=None, use_lock_file=False, attributes=(), **kwargs): |
| cmd = [ |
| 'init', |
| '-analytics-opt=false', |
| '-rewrite-sso-to-https=true', |
| '-cache', |
| ] |
| if self.m.experimental.partial_checkout: |
| cmd.append(self.m.path['cache'].join('git', 'partial')) |
| cmd.append('-partial'), |
| else: |
| cmd.append(self.m.path['cache'].join('git')) |
| if use_lock_file: |
| cmd.append('-enable-lockfile=true') |
| if attributes: |
| cmd.append('-fetch-optional=%s' % ','.join(attributes)) |
| if directory: |
| cmd.append(directory) |
| |
| return self(*cmd, **kwargs) |
| |
| def project(self, projects=(), list_remote=False, out=None, test_data=None): |
| """ |
| Args: |
| projects (List): A heterogeneous list of strings representing the name of |
| projects to list info about (defaults to all). |
| list_remote (bool): Show remote projects and include manifest paths. |
| out (str): Path to write json results to, with the following schema: |
| [ |
| { |
| "name": "zircon", |
| "path": "local/path/to/zircon", |
| "relativePath": "zircon", |
| "remote": "https://fuchsia.googlesource.com/zircon", |
| "revision": "af8fd6138748bc11d31a5bde3303cdc19c7e04e9", |
| "current_branch": "master", |
| "branches": [ |
| "master" |
| ] |
| } |
| ... |
| ] |
| If list_remote is True, "manifest": "local/path/to/manifest" will |
| be included in the schema. |
| |
| |
| Returns: |
| A step to provide structured info on existing projects and branches. |
| """ |
| cmd = [ |
| 'project', |
| '-json-output', |
| self.m.json.output(leak_to=out), |
| ] |
| if list_remote: |
| cmd.append('-list-remote-projects') |
| cmd.extend(projects) |
| |
| if test_data is None: |
| test_data = [ |
| { |
| 'name': p, |
| # Specify a path under start_dir to satisfy consumers that expect a |
| # "realistic" path, such as LUCI's PathApi.abs_to_path. |
| 'path': self.m.path.join('path', 'to', p), |
| 'remote': 'https://fuchsia.googlesource.com/' + p, |
| 'revision': 'c22471f4e3f842ae18dd9adec82ed9eb78ed1127', |
| 'current_branch': '', |
| 'branches': ['(HEAD detached at c22471f)'] |
| } for p in projects |
| ] |
| if list_remote: |
| for p_info in test_data: |
| p_info['manifest'] = self.m.path.join(p_info['name'], 'manifest') |
| |
| return self(*cmd, step_test_data=lambda: self.test_api.project(test_data)) |
| |
| def package(self, packages, out=None, test_data=None): |
| """ |
| Args: |
| packages (List): A heterogeneous list of strings representing the name of |
| packages to list info about (empty seq means all). |
| out (str): Path to write json results to, with the following schema: |
| [ |
| { |
| "name": "fuchsia/sysroot/linux-arm64", |
| "path": "local/path/to/fuchsia/sysroot/linux-arm64", |
| "manifest": "local/path/to/fuchsia/manifest", |
| "version": "git_revision:2c0292589d87f2abe69031b368f957687794eafc" |
| } |
| ... |
| ] |
| |
| |
| Returns: |
| A step to provide structured info on existing packages. |
| """ |
| cmd = [ |
| 'package', |
| '-json-output', |
| self.m.json.output(leak_to=out), |
| ] + packages |
| |
| if test_data is None: |
| test_data = [ |
| { |
| 'name': |
| p, |
| # Specify a path under start_dir to satisfy consumers that expect a |
| # "realistic" path, such as LUCI's PathApi.abs_to_path. |
| 'path': |
| str(self.m.path['start_dir'].join('path', 'to', p)), |
| 'version': |
| 'git_revision:0f3fc5584ab59ed7d53db93f5bd89d61f8fee337', |
| 'manifest': |
| str(self.m.path['start_dir'].join(p, 'manifest')) |
| } for p in packages |
| ] |
| |
| return self(*cmd, step_test_data=lambda: self.test_api.package(test_data)) |
| |
| def update(self, |
| gc=False, |
| rebase_tracked=False, |
| local_manifest=False, |
| run_hooks=True, |
| fetch_packages=True, |
| snapshot=None, |
| override_optional=False, |
| attempts=3, |
| **kwargs): |
| cmd = [ |
| 'update', |
| '-autoupdate=false', |
| '-attempts=%d' % attempts, |
| ] |
| if gc: |
| cmd.append('-gc=true') |
| if rebase_tracked: |
| cmd.append('-rebase-tracked') |
| if local_manifest: |
| cmd.append('-local-manifest=true') |
| if not run_hooks: |
| cmd.append('-run-hooks=false') |
| if not fetch_packages: |
| cmd.append('-fetch-packages=false') |
| if override_optional: |
| cmd.append('-override-optional=true') |
| if snapshot is not None: |
| cmd.append(snapshot) |
| |
| # Macs are extremely slow, giving them double what |
| # every other bots has for a timeout while we try |
| # to root cause and resolve. Context: fxb/43071 |
| if self.m.platform.is_mac and kwargs.get('timeout'): |
| kwargs['timeout'] = int(kwargs['timeout']) * 2 |
| |
| return self(*cmd, **kwargs) |
| |
| def run_hooks(self, local_manifest=False, attempts=3): |
| cmd = [ |
| 'run-hooks', |
| '-attempts=%d' % attempts, |
| ] |
| if local_manifest: |
| cmd.append('-local-manifest=true') |
| return self(*cmd) |
| |
| def fetch_packages(self, local_manifest=False, attempts=3): |
| cmd = [ |
| 'fetch-packages', |
| '-attempts=%d' % attempts, |
| ] |
| if local_manifest: |
| cmd.append('-local-manifest=true') |
| return self(*cmd) |
| |
| def clean(self, clean_all=False, **kwargs): |
| cmd = [ |
| 'project', |
| '-clean-all' if clean_all else '-clean', |
| ] |
| kwargs.setdefault('name', 'jiri project clean') |
| |
| return self(*cmd, **kwargs) |
| |
| def import_manifest(self, |
| manifest, |
| remote, |
| name=None, |
| revision=None, |
| overwrite=False, |
| remote_branch=None, |
| **kwargs): |
| """Imports manifest into Jiri project. |
| |
| Args: |
| manifest (str): A file within the repository to use. |
| remote (str): A remote manifest repository address. |
| name (str): The name of the remote manifest project. |
| revision (str): A revision to checkout for the remote. |
| remote_branch (str): A branch of the remote manifest repository |
| to checkout. If a revision is specified, this value is ignored. |
| |
| Returns: |
| A step result. |
| """ |
| cmd = ['import'] |
| if name: |
| cmd.extend(['-name', name]) |
| if overwrite: |
| cmd.extend(['-overwrite=true']) |
| |
| # revision cannot be passed along with remote-branch, because jiri. |
| if remote_branch: |
| cmd.extend(['-remote-branch', remote_branch]) |
| elif revision: |
| cmd.extend(['-revision', revision]) |
| |
| cmd.extend([manifest, remote]) |
| |
| return self(*cmd, **kwargs) |
| |
| def edit_manifest(self, |
| manifest, |
| projects=None, |
| imports=None, |
| packages=None, |
| test_data=None, |
| **kwargs): |
| """Creates a step to edit a Jiri manifest. |
| |
| Args: |
| manifest (str): Path to the manifest to edit relative to the project root. |
| e.g. "manifest/zircon" |
| projects (List): A heterogeneous list whose entries are either: |
| * A string representing the name of a project to edit. |
| * A (name, revision) tuple representing a project to edit. |
| imports (List): The same as projects, except each list entry represents an |
| import to edit. |
| packages (List): A heterogeneous list whose entries are a (name, version) |
| tuple representing a project to edit. |
| |
| Returns: |
| A step to edit the manifest. |
| """ |
| |
| cmd = [ |
| 'edit', |
| '-json-output', |
| self.m.json.output(), |
| ] |
| # Test data consisting of (name, revision) tuples of imports to edit in the |
| # given manifest. |
| test_imports = [] |
| |
| if imports: |
| for i in imports: |
| if isinstance(i, str): |
| cmd.extend(['-import', i]) |
| test_imports.append((i, 'HEAD')) |
| elif isinstance(i, tuple): |
| cmd.extend(['-import', '%s=%s' % i]) |
| test_imports.append(i) |
| |
| # Test data consisting of (name, revision) tuples of projects to edit in the |
| # given manifest. |
| test_projects = [] |
| |
| if projects: |
| for p in projects: |
| if isinstance(p, str): |
| cmd.extend(['-project', p]) |
| test_projects.append((p, 'HEAD')) |
| elif isinstance(p, tuple): |
| cmd.extend(['-project', '%s=%s' % p]) |
| test_projects.append(p) |
| |
| # Test data consisting of (name, version) tuples of packages to edit in the |
| # given manifest. |
| test_packages = [] |
| |
| if packages: |
| for p in packages: |
| cmd.extend(['-package', '%s=%s' % p]) |
| test_packages.append(p) |
| |
| cmd.extend([manifest]) |
| |
| # Generate test data |
| if test_data is None: |
| test_data = self.test_api.example_edit( |
| imports=test_imports, |
| projects=test_projects, |
| packages=test_packages, |
| ) |
| |
| step = self( |
| *cmd, step_test_data=lambda: self.test_api.project(test_data), **kwargs) |
| return step.json.output |
| |
| def patch(self, |
| ref, |
| host=None, |
| project=None, |
| delete=False, |
| force=False, |
| rebase=False, |
| rebase_revision='', |
| cherrypick=False): |
| """Patches in a CL to the current workspace. |
| |
| Args: |
| ref (str): The git ref to patch in. |
| host (str): The gerrit host from which to retrieve the ref. |
| project (str): The gerrit project (repo) from which to retrieve the ref. |
| delete (bool): Delete the existing branch if already exists. |
| force (bool): Use -force when deleting the existing branch. |
| rebase (bool): Rebase the patch on top of rebase_revision after downloading it. |
| rebase_revision (str): The revision on which to rebase. |
| cherrypick (bool): Cherrypick instead of check out. |
| |
| Raises: |
| RebaseError: If the command failed due to being unable to rebase the |
| patch on top of HEAD. |
| Returns: |
| StepData for the patch step. |
| """ |
| cmd = ['patch'] |
| if host: |
| cmd.extend(['-host', host]) |
| if project: |
| cmd.extend(['-project', project]) |
| if delete: |
| cmd.extend(['-delete=true']) |
| if force: |
| cmd.extend(['-force=true']) |
| if rebase: # pragma: no cover |
| cmd.extend(['-rebase=true']) |
| if rebase_revision: # pragma: no cover |
| cmd.extend(['-rebase-revision', rebase_revision]) |
| if cherrypick: # pragma: no cover |
| cmd.extend(['-cherry-pick=true']) |
| cmd.extend([ref]) |
| |
| step_result = self(*cmd, ok_ret=(0, JiriApi.RebaseError.EXIT_CODE)) |
| if step_result.retcode == JiriApi.RebaseError.EXIT_CODE: |
| raise JiriApi.RebaseError('Failed to rebase') |
| return step_result |
| |
| def override(self, project, remote, new_revision='HEAD'): |
| """Overrides a given project entry with a new revision. |
| |
| Args: |
| project (str): name of the project. |
| remote (str): URL to the remote repository. |
| new_revision (str or None): new revision to override the project's current. |
| """ |
| cmd = ['override', '-revision', new_revision, project, remote] |
| return self(*cmd) |
| |
| def resolve(self, |
| manifests, |
| local_manifest=False, |
| enable_project_lock=False, |
| enable_package_lock=True, |
| output=None, |
| step_test_data=None): |
| """Generate jiri lockfile in json format for <manifest ...>. If no manifest |
| provided, jiri will use .jiri_manifest by default. |
| |
| Args: |
| manifests (seq of str): manifests to generate a lock file for. |
| local_manifest (bool): whether to use local manifest. |
| enable_project_lock (bool): whether to generate locks for projects. |
| enable_package_lock (bool): whether to generate locks for packages. |
| output (Path): path to the generated lockfile. |
| step_test_data (str): test data for the stdout of this command. |
| """ |
| cmd = [ |
| 'resolve', |
| '-local-manifest=%r' % local_manifest, |
| '-enable-project-lock=%r' % enable_project_lock, |
| '-enable-package-lock=%r' % enable_package_lock, |
| ] |
| if output: |
| cmd.extend(['-output', output]) |
| if manifests: |
| cmd.extend(manifests) |
| step_result = self( |
| *cmd, |
| stdout=self.m.raw_io.output(), |
| step_test_data=lambda: self.m.raw_io.test_api.stream_output( |
| step_test_data)) |
| step_result.presentation.logs['output'] = step_result.stdout.split('\n') |
| return step_result |
| |
| def snapshot(self, snapshot_file=None, test_data=None, **kwargs): |
| cmd = [ |
| 'snapshot', |
| self.m.raw_io.output(name='snapshot', leak_to=snapshot_file), |
| ] |
| if test_data is None: |
| test_data = self.test_api.example_snapshot |
| step = self( |
| *cmd, |
| step_test_data=lambda: self.test_api.snapshot(test_data), |
| **kwargs) |
| return step.raw_io.output |
| |
| def diff(self, |
| snapshot_file_1, |
| snapshot_file_2, |
| max_cls=5, |
| out=None, |
| test_data=None, |
| **kwargs): |
| """ |
| Take a diff of two snapshots and return per-project CL changes between |
| them. |
| |
| Args: |
| snapshot_file_1 (Path): Path to first snapshot. |
| snapshot_file_2 (Path): Path to second snapshot. |
| max_cls (int): Maximum CLs per project to return. |
| out (Path): Path to write diff output to. |
| |
| Returns: |
| str: Per-project CL changes. Note the jiri diff subcommand does not |
| support -json-output and stdout can include trace output, so this |
| function cannot guarantee JSON output. |
| """ |
| cmd = [ |
| 'diff', |
| '-max-cls', |
| max_cls, |
| snapshot_file_1, |
| snapshot_file_2, |
| ] |
| if test_data is None: |
| test_data = self.test_api.example_diff |
| step = self( |
| *cmd, |
| stdout=self.m.raw_io.output(leak_to=out), |
| step_test_data=lambda: self.test_api.diff(test_data), |
| **kwargs) |
| return step.stdout |
| |
| def source_manifest(self, output_file=None, test_data=None, **kwargs): |
| """Generates a source manifest JSON file. |
| |
| Args: |
| output_file (Path): Optional output path for the source manifest. |
| |
| Returns: |
| The contents of the source manifest as a Python dictionary. |
| """ |
| cmd = [ |
| 'source-manifest', |
| self.m.json.output(name='source manifest', leak_to=output_file), |
| ] |
| if test_data is None: |
| test_data = self.test_api.example_source_manifest |
| step = self( |
| *cmd, |
| step_test_data=lambda: self.test_api.source_manifest(test_data), |
| **kwargs) |
| return step.json.output |
| |
| def emit_source_manifest(self): |
| """Emits and returns a source manifest for this build for the current jiri checkout.""" |
| manifest = self.source_manifest() |
| self.m.source_manifest.set_json_manifest('checkout', manifest) |
| return manifest |
| |
| def read_manifest_element(self, manifest, element_type, element_name): |
| """Reads information about a <project>, <import> or <package> from a |
| manifest file. |
| |
| Args: |
| manifest (str or Path): Path to the manifest file. |
| element_type (str): One of 'import', 'project' or 'package'. |
| element_name (str): The name of the element. |
| |
| Returns: |
| A dict containing the project fields. Any fields that are missing or have |
| empty values are omitted from the dict. Examples: |
| |
| # Read remote attribute of the returned project |
| print(project['remote']) # https://fuchsia.googlesource.com/my_project |
| |
| # Check whether remote attribute was present and non-empty. |
| if 'remote' in project: |
| ... |
| """ |
| if element_type == 'project': |
| # This is a Go template matching the schema in pkg/text/template. See docs |
| # for `__manifest` for more details. We format the template as JSON to |
| # make it easy to parse into a dict. The template contains the fields in |
| # a manifest <project>. See //jiri/project/manifest.go for the original |
| # definition. Add fields to this template as-needed. |
| template = """ |
| { |
| "gerrithost": "{{.GerritHost}}", |
| "githooks": "{{.GitHooks}}", |
| "historydepth": "{{.HistoryDepth}}", |
| "name": "{{.Name}}", |
| "path": "{{.Path}}", |
| "remote": "{{.Remote}}", |
| "remotebranch": "{{.RemoteBranch}}", |
| "revision": "{{.Revision}}" |
| } |
| """ |
| elif element_type == 'import': |
| # This template contains the fields in a manifest <import>. See |
| # //jiri/project/manifest.go for the original definition. |
| template = """ |
| { |
| "manifest": "{{.Manifest}}", |
| "name": "{{.Name}}", |
| "remote": "{{.Remote}}", |
| "revision": "{{.Revision}}", |
| "remotebranch": "{{.RemoteBranch}}", |
| "root": "{{.Root}}" |
| } |
| """ |
| else: |
| assert element_type == 'package' |
| # This template contains the fields in a manifest <package>. See |
| # //jiri/project/manifest.go for the original definition. |
| template = """ |
| { |
| "name": "{{.Name}}", |
| "version": "{{.Version}}", |
| "path": "{{.Path}}", |
| "internal": "{{.Internal}}", |
| "attributes": "{{.Attributes}}" |
| } |
| """ |
| # Parse the result as JSON |
| with self.m.step.nest('read_manifest_' + element_name): |
| element_json = self.__manifest( |
| manifest=manifest, |
| element_name=element_name, |
| template=template, |
| stdout=self.m.json.output(), |
| ).stdout |
| |
| # Strip whitespace from any attribute values. Discard empty values. |
| return {k: v.strip() for k, v in element_json.iteritems() if v.strip()} |
| |
| def __manifest(self, manifest, element_name, template, **kwargs): |
| """Reads information about a supported element (project, import, or package) |
| from a manifest file. |
| |
| The template argument is a Go template string matching the schema defined |
| in pkg/text/template: https://golang.org/pkg/text/template/#hdr-Examples. |
| |
| Any of the elements's fields may be specified in the template. |
| |
| See https://fuchsia.googlesource.com/jiri/+/master/project/project.go for a |
| list of all of the elements' fields. |
| |
| Example Usage: |
| |
| # Read the remote= attribute of some <project>. |
| # |
| # Example output: https://code.com/my_project.git |
| __manifest(manifest='manifest', element_name='my_project', |
| template='{{.Remote}}') |
| |
| # Read the remote= and path= attributes from some <import>, and |
| # format the output as "$remote is cloned to $path". |
| # |
| # Example output: https://code.com/my_import.git is cloned to /my_import. |
| __manifest(manifest='manifest', element_name='my_import', |
| template='{{.Remote}}) is cloned to {{.Path}}') |
| |
| Args: |
| manifest (str or Path): Path to the manifest file. |
| element_name (str): The name of the element to read from. |
| template (str): A Go template string matching pkg/text/template. |
| |
| Returns: |
| The filled-in template string. If the element did not have a value for |
| some field in the template, the empty string is filled-in for that |
| field. |
| """ |
| return self('manifest', '-element', element_name, '-template', template, |
| manifest, **kwargs) |
