| # Copyright 2016 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from recipe_engine import recipe_api |
| |
| |
| class JiriApi(recipe_api.RecipeApi): |
| """JiriApi provides support for Jiri managed checkouts.""" |
| |
| # If a patch fails because the specified project is not in the |
| # checkout, it probably means someone triggered a builder on a CL |
| # that's not part of the builder's checkout. This is not an infra |
| # issue. |
| class NoSuchProjectError(recipe_api.StepFailure): |
| # Keep in sync with |
| # https://fuchsia.googlesource.com/jiri/+/main/cmd/jiri/patch.go |
| EXIT_CODE = 23 |
| |
| # A failure to rebase is closer to user error than an infra failure. |
| # It can be fixed by the user by rebasing their change and then |
| # retrying the patch. The infrastructure is working properly. |
| class RebaseError(recipe_api.StepFailure): |
| # Keep in sync with |
| # https://fuchsia.googlesource.com/jiri/+/main/cmd/jiri/patch.go |
| EXIT_CODE = 24 |
| |
| # A failure in `cipd verify-ensure-file` indicates a missing or |
| # malformed package entry. This most often represents a user error |
| # when modifying jiri manifests in integration. |
| class EnsureFileVerificationError(recipe_api.StepFailure): |
| EXIT_CODE = 25 |
| |
| @property |
| def _common_flags(self): |
| # Flags added to all jiri commands. |
| return ["-vv", "-time", f"-j={int(self.m.platform.cpu_count)}"] |
| |
| @property |
| def jiri(self): |
| return self.m.ensure_tool("jiri", self.resource("tool_manifest.json")) |
| |
| def init( |
| self, |
| directory=None, |
| dissociate=False, |
| enable_submodules=False, |
| use_lock_file=False, |
| attributes=(), |
| **kwargs, |
| ): |
| """ |
| Args: |
| directory (str): The directory to init a .jiri_root/ in. |
| enable_submodules (bool): Whether to enable submodules. |
| use_lock_file (bool): Whether to enforce lock files in the jiri root. |
| attributes (seq(str)): A list of attributes used to identify optional |
| projects to be added to the checkout. |
| |
| Returns: |
| A step to provide structured info on existing projects and branches. |
| """ |
| cmd = [ |
| "init", |
| "-analytics-opt=false", |
| "-rewrite-sso-to-https=true", |
| "-cache", |
| self.m.path["cache"].join("git"), |
| f"-fetch-optional={','.join(attributes)}", |
| ] |
| if dissociate: |
| cmd.append("-dissociate=true") |
| if use_lock_file: |
| cmd.append("-enable-lockfile=true") |
| if enable_submodules: |
| cmd.append("-enable-submodules=true") |
| if directory: |
| cmd.append(directory) |
| |
| return self._run(*cmd, **kwargs) |
| |
| def project( |
| self, projects=(), list_remote=False, out=None, test_data=None, **kwargs |
| ): |
| """ |
| Args: |
| projects (List): A heterogeneous list of strings representing the name of |
| projects to list info about (defaults to all). |
| list_remote (bool): Show remote projects and include manifest paths. |
| out (str): Path to write json results to, with the following schema: |
| [ |
| { |
| "name": "foo", |
| "path": "/absolute/path/to/checkout/foo", |
| "relativePath": "foo", |
| "remote": "https://fuchsia.googlesource.com/foo", |
| "revision": "af8fd6138748bc11d31a5bde3303cdc19c7e04e9", |
| "current_branch": "main", |
| "branches": [ |
| "main" |
| ] |
| } |
| ... |
| ] |
| If list_remote is True, "manifest": "local/path/to/manifest" will |
| be included in the schema. |
| |
| Returns: |
| A step to provide structured info on existing projects and branches. |
| """ |
| cmd = [ |
| "project", |
| "-json-output", |
| self.m.json.output(leak_to=out), |
| ] |
| if list_remote: |
| cmd.append("-list-remote-projects") |
| cmd.extend(projects) |
| |
| if test_data is None: |
| cwd = self.m.context.cwd or self.m.path["start_dir"] |
| test_data = [ |
| { |
| "name": p, |
| # Specify a path under start_dir to satisfy consumers that expect a |
| # "realistic" path, such as LUCI's PathApi.abs_to_path. |
| "path": str(cwd.join(p)), |
| "relativePath": p, |
| "remote": "https://fuchsia.googlesource.com/" + p, |
| "revision": "c22471f4e3f842ae18dd9adec82ed9eb78ed1127", |
| "current_branch": "", |
| "branches": ["(HEAD detached at c22471f)"], |
| } |
| for p in projects |
| ] |
| if list_remote: |
| for p_info in test_data: |
| p_info["manifest"] = self.m.path.join(p_info["name"], "manifest") |
| |
| return self._run( |
| *cmd, step_test_data=lambda: self.test_api.project(test_data), **kwargs |
| ) |
| |
| def package(self, packages, out=None, test_data=None): |
| """ |
| Args: |
| packages (List): A heterogeneous list of strings representing the name of |
| packages to list info about (empty seq means all). |
| out (str): Path to write json results to, with the following schema: |
| [ |
| { |
| "name": "fuchsia/foo/linux-arm64", |
| "path": "local/path/to/fuchsia/foo/linux-arm64", |
| "manifest": "local/path/to/fuchsia/manifest", |
| "version": "git_revision:2c0292589d87f2abe69031b368f957687794eafc" |
| } |
| ... |
| ] |
| |
| Returns: |
| A step to provide structured info on existing packages. |
| """ |
| cmd = ["package", "-json-output", self.m.json.output(leak_to=out)] |
| cmd.extend(packages) |
| |
| if test_data is None: |
| test_data = [ |
| { |
| "name": p, |
| # Specify a path under start_dir to satisfy consumers that expect a |
| # "realistic" path, such as LUCI's PathApi.abs_to_path. |
| "path": str(self.m.path["start_dir"].join("path", "to", p)), |
| "version": "git_revision:0f3fc5584ab59ed7d53db93f5bd89d61f8fee337", |
| "manifest": str(self.m.path["start_dir"].join(p, "manifest")), |
| } |
| for p in packages |
| ] |
| |
| step = self._run(*cmd, step_test_data=lambda: self.test_api.package(test_data)) |
| |
| # Jiri will silently not return any packages that don't exist in the |
| # checkout. Turn this into a loud failure. |
| missing = set(packages) - {p["name"] for p in step.json.output} |
| if missing: |
| step.presentation.step_text = f"some packages were not found: {missing}" |
| step.presentation.status = self.m.step.INFRA_FAILURE |
| self.m.step.raise_on_failure(step) |
| |
| return step |
| |
| def reset(self): |
| """Checkout JIRI_HEAD in all repos.""" |
| cmd = [ |
| "runp", |
| "--", |
| "git", |
| "checkout", |
| "--detach", |
| "JIRI_HEAD", |
| ] |
| |
| return self._run(*cmd) |
| |
| def check_clean(self): |
| """Check whether all repos in the checkout are in a clean state.""" |
| cmd = ["check-clean"] |
| return self._run(*cmd) |
| |
| def update( |
| self, |
| gc=False, |
| rebase_tracked=False, |
| local_manifest=False, |
| run_hooks=True, |
| fetch_packages=True, |
| snapshot=None, |
| override_optional=False, |
| packages_to_skip=(), |
| **kwargs, |
| ): |
| cmd = [ |
| "update", |
| "-autoupdate=false", |
| "-attempts=1", |
| ] |
| env = {} |
| if gc: |
| cmd.append("-gc=true") |
| if rebase_tracked: |
| cmd.append("-rebase-tracked") |
| if local_manifest: |
| cmd.append("-local-manifest=true") |
| if not run_hooks: |
| cmd.append("-run-hooks=false") |
| else: |
| env.update(self.runhooks_env) |
| if not fetch_packages: |
| cmd.append("-fetch-packages=false") |
| else: |
| cmd.extend([f"-package-to-skip={p}" for p in packages_to_skip]) |
| if override_optional: |
| cmd.append("-override-optional=true") |
| if snapshot is not None: |
| cmd.append(snapshot) |
| |
| with self.m.cache.guard(cache="git"): |
| return self._run(*cmd, **kwargs) |
| |
| def run_hooks( |
| self, |
| local_manifest=False, |
| fetch_packages=True, |
| hook_timeout_secs=None, |
| packages_to_skip=(), |
| attempts=3, |
| ): |
| cmd = [ |
| "run-hooks", |
| f"-attempts={int(attempts)}", |
| ] |
| # Note: when fetching packages, Jiri sets the timeout to 5x the hook |
| # timeout. The flag expects a value in minutes. |
| if hook_timeout_secs: |
| cmd.append(f"-hook-timeout={int(hook_timeout_secs / 60)}") |
| if local_manifest: |
| cmd.append("-local-manifest=true") |
| if not fetch_packages: |
| cmd.append("-fetch-packages=false") |
| else: |
| cmd.extend([f"-package-to-skip={p}" for p in packages_to_skip]) |
| with self.m.context(env=self.runhooks_env): |
| step_result = self._run( |
| *cmd, ok_ret=(0, self.EnsureFileVerificationError.EXIT_CODE) |
| ) |
| if step_result.retcode == self.EnsureFileVerificationError.EXIT_CODE: |
| raise self.EnsureFileVerificationError("bad cipd package entry") |
| return step_result |
| |
| def fetch_packages(self, local_manifest=False, packages_to_skip=(), attempts=3): |
| cmd = [ |
| "fetch-packages", |
| f"-attempts={int(attempts)}", |
| ] |
| if local_manifest: |
| cmd.append("-local-manifest=true") |
| cmd.extend([f"-package-to-skip={p}" for p in packages_to_skip]) |
| step_result = self._run( |
| *cmd, ok_ret=(0, self.EnsureFileVerificationError.EXIT_CODE) |
| ) |
| if step_result.retcode == self.EnsureFileVerificationError.EXIT_CODE: |
| raise self.EnsureFileVerificationError("bad cipd package entry") |
| return step_result |
| |
| def import_manifest( |
| self, |
| manifest, |
| remote, |
| name=None, |
| revision=None, |
| overwrite=False, |
| remote_branch=None, |
| **kwargs, |
| ): |
| """Imports manifest into Jiri project. |
| |
| Args: |
| manifest (str): A file within the repository to use. |
| remote (str): A remote manifest repository address. |
| name (str): The name of the remote manifest project. |
| revision (str): A revision to checkout for the remote. |
| remote_branch (str): A branch of the remote manifest repository |
| to checkout. If a revision is specified, this value is ignored. |
| |
| Returns: |
| A step result. |
| """ |
| cmd = ["import"] |
| if name: |
| cmd.extend(["-name", name]) |
| if overwrite: |
| cmd.extend(["-overwrite=true"]) |
| |
| # revision cannot be passed along with remote-branch, because jiri. |
| if remote_branch: |
| cmd.extend(["-remote-branch", remote_branch]) |
| elif revision: |
| cmd.extend(["-revision", revision]) |
| |
| cmd.extend([manifest, remote]) |
| |
| return self._run(*cmd, **kwargs) |
| |
| def get_import_revision( |
| self, import_name, manifest=".jiri_manifest", test_data=None, **kwargs |
| ): |
| """Return the git revision of the given import. |
| |
| Args: |
| import_name (str): The name of the import to get the version of. |
| manifest (str): The path to the manifest, relative to jiri root. |
| kwargs (dict): Passed through to api.step(). |
| |
| Returns: |
| A git revision, or None if `ok_ret` was set and the command failed |
| (e.g. because it was run outside a Jiri root). |
| """ |
| cmd = [ |
| "manifest", |
| "-element", |
| import_name, |
| "-template={{.Revision}}", |
| manifest, |
| ] |
| step = self._run( |
| *cmd, |
| stdout=self.m.raw_io.output_text(), |
| step_test_data=lambda: self.m.raw_io.test_api.stream_output_text(test_data), |
| **kwargs, |
| ) |
| if step.retcode: |
| return None |
| return step.stdout.strip() |
| |
| def edit_manifest( |
| self, |
| manifest, |
| projects=(), |
| packages=(), |
| test_data=None, |
| **kwargs, |
| ): |
| """Creates a step to edit a Jiri manifest. |
| |
| Args: |
| manifest (str): Path to the manifest to edit relative to the project root. |
| e.g. "manifest/zircon" |
| projects (List): A heterogeneous list whose entries are either: |
| * A string representing the name of a project to edit. |
| * A (name, revision) tuple representing a project to edit. |
| packages (List): A heterogeneous list whose entries are a (name, version) |
| tuple representing a project to edit. |
| |
| Returns: |
| A step to edit the manifest. |
| """ |
| |
| cmd = [ |
| "edit", |
| "-json-output", |
| self.m.json.output(), |
| ] |
| |
| # Test data consisting of (name, revision) tuples of projects to edit in the |
| # given manifest. |
| test_projects = [] |
| for p in projects: |
| if isinstance(p, str): |
| cmd.extend(["-project", p]) |
| test_projects.append((p, "HEAD")) |
| elif isinstance(p, tuple): |
| cmd.extend(["-project", f"{p[0]}={p[1]}"]) |
| test_projects.append(p) |
| |
| # Test data consisting of (name, version) tuples of packages to edit in the |
| # given manifest. |
| test_packages = [] |
| for p in packages: |
| cmd.extend(["-package", f"{p[0]}={p[1]}"]) |
| test_packages.append(p) |
| |
| cmd.extend([manifest]) |
| |
| if test_data is None: |
| test_data = self.test_api.example_edit( |
| projects=test_projects, |
| packages=test_packages, |
| ) |
| |
| return self._run( |
| *cmd, step_test_data=lambda: self.test_api.project(test_data), **kwargs |
| ).json.output |
| |
| def patch( |
| self, |
| ref, |
| host=None, |
| project=None, |
| delete=True, |
| force=True, |
| rebase=False, |
| rebase_branch="", |
| rebase_revision="", |
| ): |
| """Patches in a CL to the current workspace. |
| |
| Args: |
| ref (str): The git ref to patch in. |
| host (str): The gerrit host from which to retrieve the ref. |
| project (str): The gerrit project (repo) from which to retrieve the ref. |
| delete (bool): Delete the existing branch if already exists. |
| force (bool): Use -force when deleting the existing branch. |
| rebase (bool): Rebase the patch on top of rebase_branch or |
| rebase_revision after downloading it. |
| rebase_branch (str): The branch on which to rebase. |
| rebase_revision (str): The revision on which to rebase. |
| |
| Raises: |
| RebaseError: If the command failed due to being unable to rebase the |
| patch on top of HEAD. |
| Returns: |
| StepData for the patch step. |
| """ |
| cmd = ["patch"] |
| if host: |
| cmd.extend(["-host", host]) |
| if project: |
| cmd.extend(["-project", project]) |
| if delete: |
| cmd.extend(["-delete=true"]) |
| if force: |
| cmd.extend(["-force=true"]) |
| if rebase: # pragma: no cover |
| cmd.extend(["-rebase=true"]) |
| if rebase_branch: # pragma: no cover |
| cmd.extend(["-rebase-branch", rebase_branch]) |
| if rebase_revision: # pragma: no cover |
| cmd.extend(["-rebase-revision", rebase_revision]) |
| cmd.extend([ref]) |
| |
| step_result = self._run( |
| *cmd, |
| ok_ret=(0, self.NoSuchProjectError.EXIT_CODE, self.RebaseError.EXIT_CODE), |
| ) |
| if step_result.retcode == self.NoSuchProjectError.EXIT_CODE: |
| raise self.NoSuchProjectError( |
| f"Project {project} is not part of the checkout" |
| ) |
| elif step_result.retcode == self.RebaseError.EXIT_CODE: |
| raise self.RebaseError("Failed to rebase") |
| return step_result |
| |
| def override(self, project, remote, new_revision="HEAD"): |
| """Overrides a given project entry with a new revision. |
| |
| Args: |
| project (str): name of the project. |
| remote (str): URL to the remote repository. |
| new_revision (str or None): new revision to override the project's current. |
| """ |
| cmd = ["override", "-revision", new_revision, project, remote] |
| return self._run(*cmd) |
| |
| def resolve( |
| self, |
| manifests, |
| local_manifest=False, |
| enable_project_lock=False, |
| enable_package_lock=True, |
| output=None, |
| step_test_data=None, |
| ): |
| """Generate jiri lockfile in json format for <manifest ...>. |
| |
| If no manifest provided, jiri will use .jiri_manifest by default. |
| |
| Args: |
| manifests (seq of str): manifests to generate a lock file for. |
| local_manifest (bool): whether to use local manifest. |
| enable_project_lock (bool): whether to generate locks for projects. |
| enable_package_lock (bool): whether to generate locks for packages. |
| output (Path): path to the generated lockfile. |
| step_test_data (str): test data for the stdout of this command. |
| """ |
| cmd = [ |
| "resolve", |
| f"-local-manifest={local_manifest!r}", |
| f"-enable-project-lock={enable_project_lock!r}", |
| f"-enable-package-lock={enable_package_lock!r}", |
| ] |
| if output: |
| cmd.extend(["-output", output]) |
| if manifests: |
| cmd.extend(manifests) |
| step_result = self._run( |
| *cmd, |
| stdout=self.m.raw_io.output_text(), |
| step_test_data=lambda: self.m.raw_io.test_api.stream_output_text( |
| step_test_data |
| ), |
| ) |
| step_result.presentation.logs["output"] = step_result.stdout.split("\n") |
| return step_result |
| |
| def snapshot(self, snapshot_file, test_data=None, cipd=False, **kwargs): |
| """Generates snapshot that captures the current project state in a manifest. |
| |
| Args: |
| snapshot_file (Path): Required output path for the snapshot file. |
| cipd (bool): Whether to generate additional cipd ensure and version snapshots. |
| |
| Returns: |
| A dict mapping file name to snapshot, package ensure and version files. |
| "snapshot": location of snapshot file. |
| "ensure": external package ensure file. |
| "version": external package version file. |
| "internal_ensure": internal package ensure file. |
| "internal_version": internal package version file. |
| """ |
| cmd = ["snapshot"] |
| if cipd: |
| cmd.append("-cipd") |
| cmd.append(self.m.raw_io.output_text(name="snapshot", leak_to=snapshot_file)) |
| if test_data is None: |
| test_data = self.test_api.example_snapshot |
| self._run( |
| *cmd, step_test_data=lambda: self.test_api.snapshot(test_data), **kwargs |
| ) |
| snapshot_files = {"snapshot": snapshot_file} |
| if cipd: |
| dirname, basename = self.m.path.split(snapshot_file) |
| snapshot_files["ensure"] = dirname.join(basename + ".ensure") |
| snapshot_files["version"] = dirname.join(basename + ".version") |
| snapshot_files["internal_ensure"] = dirname.join( |
| basename + "_internal.ensure" |
| ) |
| snapshot_files["internal_version"] = dirname.join( |
| basename + "_internal.version" |
| ) |
| |
| return snapshot_files |
| |
| def snapshot_diff( |
| self, snapshot_1, snapshot_2, test_data=None, output_file=None, **kwargs |
| ): |
| """Generates snapshot diff file between two snapshots. |
| |
| Args: |
| snapshot_1 (Path): File or url containing snapshot. |
| snapshot_2 (Path): File or url containing snapshot. |
| output_file (Path): Optional output path for the json diff file. |
| |
| Returns: |
| Python dict of the snapshot diff between two snapshots in json format. |
| Max CLs returned for a project is controlled by flag max-cls and is 5 |
| by default. |
| """ |
| cmd = [ |
| "diff", |
| snapshot_1, |
| snapshot_2, |
| ] |
| if test_data is None: |
| test_data = self.test_api.example_snapshot_diff |
| step = self._run( |
| *cmd, |
| stdout=self.m.json.output(name="snapshot diff", leak_to=output_file), |
| step_test_data=lambda: self.test_api.snapshot_diff(test_data), |
| **kwargs, |
| ) |
| return step.stdout |
| |
| def source_manifest(self, output_file=None, test_data=None, **kwargs): |
| """Generates a source manifest JSON file. |
| |
| Args: |
| output_file (Path): Optional output path for the source manifest. |
| |
| Returns: |
| The contents of the source manifest as a Python dictionary. |
| """ |
| cmd = [ |
| "source-manifest", |
| self.m.json.output(name="source manifest", leak_to=output_file), |
| ] |
| if test_data is None: |
| test_data = self.test_api.example_source_manifest |
| step = self._run( |
| *cmd, |
| step_test_data=lambda: self.test_api.source_manifest(test_data), |
| **kwargs, |
| ) |
| return step.json.output |
| |
| def read_manifest_element( |
| self, manifest, element_type, element_name, name=None, **kwargs |
| ): |
| """Reads information about a <project> or <package> from a manifest file. |
| |
| Args: |
| manifest (str or Path): Path to the manifest file. |
| element_type (str): One of 'project' or 'package'. |
| element_name (str): The name of the element. |
| name (str): The name of the step. |
| kwargs (dict): Passed through to _run. |
| |
| Returns: |
| A dict containing the project fields. Any fields that are missing or have |
| empty values are omitted from the dict. Examples: |
| |
| # Read remote attribute of the returned project |
| print(project['remote']) # https://fuchsia.googlesource.com/my_project |
| |
| # Check whether remote attribute was present and non-empty. |
| if 'remote' in project: |
| ... |
| """ |
| if element_type == "project": |
| # This is a dict that becomes a Go template string when dumped to a |
| # string. we format the template as JSON to make it easy to parse |
| # into a dict. The template contains the fields in a manifest |
| # <project>. See //jiri/project/manifest.go for the original |
| # definition. |
| template_json = { |
| "gerrithost": "{{.GerritHost}}", |
| "githooks": "{{.GitHooks}}", |
| "historydepth": "{{.HistoryDepth}}", |
| "name": "{{.Name}}", |
| "path": "{{.Path}}", |
| "remote": "{{.Remote}}", |
| "remotebranch": "{{.RemoteBranch}}", |
| "revision": "{{.Revision}}", |
| } |
| else: |
| assert element_type == "package" |
| template_json = { |
| "name": "{{.Name}}", |
| "version": "{{.Version}}", |
| "path": "{{.Path}}", |
| "internal": "{{.Internal}}", |
| "attributes": "{{.Attributes}}", |
| "platforms": "{{.Platforms}}", |
| } |
| |
| if not name: |
| name = f"read manifest for {element_name}" |
| kwargs.setdefault( |
| "step_test_data", lambda: self.m.json.test_api.output_stream({}) |
| ) |
| element_json = self._run( |
| "manifest", |
| "-element", |
| element_name, |
| "-template", |
| self.m.json.dumps(template_json, indent=2), |
| manifest, |
| name=name, |
| stdout=self.m.json.output(), |
| **kwargs, |
| ).stdout |
| |
| # Strip whitespace from any attribute values and discard empty values. |
| return {k: v.strip() for k, v in element_json.items() if v.strip()} |
| |
| def generate_gitmodules( |
| self, gitmodules_path, generate_script=None, redirect_root=False |
| ): |
| cmd = ["generate-gitmodules"] |
| if redirect_root: |
| cmd.append("--redir-root") |
| if generate_script: |
| cmd.extend(["--generate-script", generate_script]) |
| cmd.append(gitmodules_path) |
| return self._run(*cmd) |
| |
| @property |
| def runhooks_env(self): |
| """Returns env so Jiri hooks can behave differently in infrastructure. |
| |
| See http://fxb/59015 for why $HOME is overridden. |
| """ |
| assert self.m.context.cwd |
| return { |
| "HOME": self.m.context.cwd, |
| "INFRA_RECIPES": "1", |
| } |
| |
| def _run(self, *args, **kwargs): |
| """Return a jiri command step.""" |
| subcommand = args[0] # E.g., 'init' or 'update' |
| flags = self._common_flags + list(args[1:]) |
| |
| full_cmd = [self.jiri, subcommand] + flags |
| |
| name = kwargs.pop("name", "jiri " + subcommand) |
| return self.m.step(name, full_cmd, **kwargs) |