blob: 96fccfc04c702511c7af2a9db95433afa162c9d1 [file] [log] [blame]
# Copyright 2016 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from recipe_engine import recipe_api
class JiriApi(recipe_api.RecipeApi):
"""JiriApi provides support for Jiri managed checkouts."""
# If a patch fails because the specified project is not in the
# checkout, it probably means someone triggered a builder on a CL
# that's not part of the builder's checkout. This is not an infra
# issue.
class NoSuchProjectError(recipe_api.StepFailure):
# Keep in sync with
# https://fuchsia.googlesource.com/jiri/+/main/cmd/jiri/patch.go
EXIT_CODE = 23
# A failure to rebase is closer to user error than an infra failure.
# It can be fixed by the user by rebasing their change and then
# retrying the patch. The infrastructure is working properly.
class RebaseError(recipe_api.StepFailure):
# Keep in sync with
# https://fuchsia.googlesource.com/jiri/+/main/cmd/jiri/patch.go
EXIT_CODE = 24
# A failure in `cipd verify-ensure-file` indicates a missing or
# malformed package entry. This most often represents a user error
# when modifying jiri manifests in integration.
class EnsureFileVerificationError(recipe_api.StepFailure):
EXIT_CODE = 25
@property
def _common_flags(self):
# Flags added to all jiri commands.
return ["-vv", "-time", "-j=%d" % self.m.platform.cpu_count]
@property
def jiri(self):
return self.m.ensure_tool("jiri", self.resource("tool_manifest.json"))
def init(
self,
directory=None,
enable_submodules=False,
use_lock_file=False,
attributes=(),
**kwargs,
):
"""
Args:
directory (str): The directory to init a .jiri_root/ in.
enable_submodules (bool): Whether to enable submodules.
use_lock_file (bool): Whether to enforce lock files in the jiri root.
attributes (seq(str)): A list of attributes used to identify optional
projects to be added to the checkout.
Returns:
A step to provide structured info on existing projects and branches.
"""
cmd = [
"init",
"-analytics-opt=false",
"-rewrite-sso-to-https=true",
"-cache",
self.m.path["cache"].join("git"),
"-fetch-optional=%s" % ",".join(attributes),
]
if use_lock_file:
cmd.append("-enable-lockfile=true")
if enable_submodules:
cmd.append("-enable-submodules=true")
if directory:
cmd.append(directory)
return self._run(*cmd, **kwargs)
def project(
self, projects=(), list_remote=False, out=None, test_data=None, **kwargs
):
"""
Args:
projects (List): A heterogeneous list of strings representing the name of
projects to list info about (defaults to all).
list_remote (bool): Show remote projects and include manifest paths.
out (str): Path to write json results to, with the following schema:
[
{
"name": "foo",
"path": "/absolute/path/to/checkout/foo",
"relativePath": "foo",
"remote": "https://fuchsia.googlesource.com/foo",
"revision": "af8fd6138748bc11d31a5bde3303cdc19c7e04e9",
"current_branch": "main",
"branches": [
"main"
]
}
...
]
If list_remote is True, "manifest": "local/path/to/manifest" will
be included in the schema.
Returns:
A step to provide structured info on existing projects and branches.
"""
cmd = [
"project",
"-json-output",
self.m.json.output(leak_to=out),
]
if list_remote:
cmd.append("-list-remote-projects")
cmd.extend(projects)
if test_data is None:
cwd = self.m.context.cwd or self.m.path["start_dir"]
test_data = [
{
"name": p,
# Specify a path under start_dir to satisfy consumers that expect a
# "realistic" path, such as LUCI's PathApi.abs_to_path.
"path": str(cwd.join(p)),
"relativePath": p,
"remote": "https://fuchsia.googlesource.com/" + p,
"revision": "c22471f4e3f842ae18dd9adec82ed9eb78ed1127",
"current_branch": "",
"branches": ["(HEAD detached at c22471f)"],
}
for p in projects
]
if list_remote:
for p_info in test_data:
p_info["manifest"] = self.m.path.join(p_info["name"], "manifest")
return self._run(
*cmd, step_test_data=lambda: self.test_api.project(test_data), **kwargs
)
def package(self, packages, out=None, test_data=None):
"""
Args:
packages (List): A heterogeneous list of strings representing the name of
packages to list info about (empty seq means all).
out (str): Path to write json results to, with the following schema:
[
{
"name": "fuchsia/sysroot/linux-arm64",
"path": "local/path/to/fuchsia/sysroot/linux-arm64",
"manifest": "local/path/to/fuchsia/manifest",
"version": "git_revision:2c0292589d87f2abe69031b368f957687794eafc"
}
...
]
Returns:
A step to provide structured info on existing packages.
"""
cmd = ["package", "-json-output", self.m.json.output(leak_to=out)]
cmd.extend(packages)
if test_data is None:
test_data = [
{
"name": p,
# Specify a path under start_dir to satisfy consumers that expect a
# "realistic" path, such as LUCI's PathApi.abs_to_path.
"path": str(self.m.path["start_dir"].join("path", "to", p)),
"version": "git_revision:0f3fc5584ab59ed7d53db93f5bd89d61f8fee337",
"manifest": str(self.m.path["start_dir"].join(p, "manifest")),
}
for p in packages
]
return self._run(*cmd, step_test_data=lambda: self.test_api.package(test_data))
def reset(self):
"""Checkout JIRI_HEAD in all repos."""
cmd = [
"runp",
"--",
"git",
"checkout",
"--detach",
"JIRI_HEAD",
]
return self._run(*cmd)
def check_clean(self):
"""Check whether all repos in the checkout are in a clean state."""
cmd = ["check-clean"]
return self._run(*cmd)
def update(
self,
gc=False,
rebase_tracked=False,
local_manifest=False,
run_hooks=True,
fetch_packages=True,
snapshot=None,
override_optional=False,
packages_to_skip=(),
**kwargs,
):
cmd = [
"update",
"-autoupdate=false",
"-attempts=1",
]
env = {}
if gc:
cmd.append("-gc=true")
if rebase_tracked:
cmd.append("-rebase-tracked")
if local_manifest:
cmd.append("-local-manifest=true")
if not run_hooks:
cmd.append("-run-hooks=false")
else:
env.update(self.runhooks_env)
if not fetch_packages:
cmd.append("-fetch-packages=false")
else:
cmd.extend(["-package-to-skip=%s" % p for p in packages_to_skip])
if override_optional:
cmd.append("-override-optional=true")
if snapshot is not None:
cmd.append(snapshot)
with self.m.cache.guard(cache="git"):
return self._run(*cmd, **kwargs)
def run_hooks(
self,
local_manifest=False,
fetch_packages=True,
hook_timeout_secs=None,
packages_to_skip=(),
attempts=3,
):
cmd = [
"run-hooks",
"-attempts=%d" % attempts,
]
# Note: when fetching packages, Jiri sets the timeout to 5x the hook
# timeout. The flag expects a value in minutes.
if hook_timeout_secs:
cmd.append("-hook-timeout=%d" % (hook_timeout_secs / 60))
if local_manifest:
cmd.append("-local-manifest=true")
if not fetch_packages:
cmd.append("-fetch-packages=false")
else:
cmd.extend(["-package-to-skip=%s" % p for p in packages_to_skip])
with self.m.context(env=self.runhooks_env):
step_result = self._run(
*cmd, ok_ret=(0, self.EnsureFileVerificationError.EXIT_CODE)
)
if step_result.retcode == self.EnsureFileVerificationError.EXIT_CODE:
raise self.EnsureFileVerificationError("bad cipd package entry")
return step_result
def fetch_packages(self, local_manifest=False, packages_to_skip=(), attempts=3):
cmd = [
"fetch-packages",
"-attempts=%d" % attempts,
]
if local_manifest:
cmd.append("-local-manifest=true")
cmd.extend(["-package-to-skip=%s" % p for p in packages_to_skip])
step_result = self._run(
*cmd, ok_ret=(0, self.EnsureFileVerificationError.EXIT_CODE)
)
if step_result.retcode == self.EnsureFileVerificationError.EXIT_CODE:
raise self.EnsureFileVerificationError("bad cipd package entry")
return step_result
def import_manifest(
self,
manifest,
remote,
name=None,
revision=None,
overwrite=False,
remote_branch=None,
**kwargs,
):
"""Imports manifest into Jiri project.
Args:
manifest (str): A file within the repository to use.
remote (str): A remote manifest repository address.
name (str): The name of the remote manifest project.
revision (str): A revision to checkout for the remote.
remote_branch (str): A branch of the remote manifest repository
to checkout. If a revision is specified, this value is ignored.
Returns:
A step result.
"""
cmd = ["import"]
if name:
cmd.extend(["-name", name])
if overwrite:
cmd.extend(["-overwrite=true"])
# revision cannot be passed along with remote-branch, because jiri.
if remote_branch:
cmd.extend(["-remote-branch", remote_branch])
elif revision:
cmd.extend(["-revision", revision])
cmd.extend([manifest, remote])
return self._run(*cmd, **kwargs)
def get_import_revision(
self, import_name, manifest=".jiri_manifest", test_data=None, **kwargs
):
"""Return the git revision of the given import.
Args:
import_name (str): The name of the import to get the version of.
manifest (str): The path to the manifest, relative to jiri root.
kwargs (dict): Passed through to api.step().
Returns:
A git revision, or None if `ok_ret` was set and the command failed
(e.g. because it was run outside a Jiri root).
"""
cmd = [
"manifest",
"-element",
import_name,
"-template={{.Revision}}",
manifest,
]
step = self._run(
*cmd,
stdout=self.m.raw_io.output_text(),
step_test_data=lambda: self.m.raw_io.test_api.stream_output_text(test_data),
**kwargs,
)
if step.retcode:
return None
return step.stdout.strip()
def edit_manifest(
self,
manifest,
projects=None,
imports=None,
packages=None,
test_data=None,
**kwargs,
):
"""Creates a step to edit a Jiri manifest.
Args:
manifest (str): Path to the manifest to edit relative to the project root.
e.g. "manifest/zircon"
projects (List): A heterogeneous list whose entries are either:
* A string representing the name of a project to edit.
* A (name, revision) tuple representing a project to edit.
imports (List): The same as projects, except each list entry represents an
import to edit.
packages (List): A heterogeneous list whose entries are a (name, version)
tuple representing a project to edit.
Returns:
A step to edit the manifest.
"""
cmd = [
"edit",
"-json-output",
self.m.json.output(),
]
# Test data consisting of (name, revision) tuples of imports to edit in the
# given manifest.
test_imports = []
if imports:
for i in imports:
if isinstance(i, str):
cmd.extend(["-import", i])
test_imports.append((i, "HEAD"))
elif isinstance(i, tuple):
cmd.extend(["-import", "%s=%s" % i])
test_imports.append(i)
# Test data consisting of (name, revision) tuples of projects to edit in the
# given manifest.
test_projects = []
if projects:
for p in projects:
if isinstance(p, str):
cmd.extend(["-project", p])
test_projects.append((p, "HEAD"))
elif isinstance(p, tuple):
cmd.extend(["-project", "%s=%s" % p])
test_projects.append(p)
# Test data consisting of (name, version) tuples of packages to edit in the
# given manifest.
test_packages = []
if packages:
for p in packages:
cmd.extend(["-package", "%s=%s" % p])
test_packages.append(p)
cmd.extend([manifest])
# Generate test data
if test_data is None:
test_data = self.test_api.example_edit(
imports=test_imports,
projects=test_projects,
packages=test_packages,
)
step = self._run(
*cmd, step_test_data=lambda: self.test_api.project(test_data), **kwargs
)
return step.json.output
def patch(
self,
ref,
host=None,
project=None,
delete=True,
force=True,
rebase=False,
rebase_branch="",
rebase_revision="",
):
"""Patches in a CL to the current workspace.
Args:
ref (str): The git ref to patch in.
host (str): The gerrit host from which to retrieve the ref.
project (str): The gerrit project (repo) from which to retrieve the ref.
delete (bool): Delete the existing branch if already exists.
force (bool): Use -force when deleting the existing branch.
rebase (bool): Rebase the patch on top of rebase_branch or
rebase_revision after downloading it.
rebase_branch (str): The branch on which to rebase.
rebase_revision (str): The revision on which to rebase.
Raises:
RebaseError: If the command failed due to being unable to rebase the
patch on top of HEAD.
Returns:
StepData for the patch step.
"""
cmd = ["patch"]
if host:
cmd.extend(["-host", host])
if project:
cmd.extend(["-project", project])
if delete:
cmd.extend(["-delete=true"])
if force:
cmd.extend(["-force=true"])
if rebase: # pragma: no cover
cmd.extend(["-rebase=true"])
if rebase_branch: # pragma: no cover
cmd.extend(["-rebase-branch", rebase_branch])
if rebase_revision: # pragma: no cover
cmd.extend(["-rebase-revision", rebase_revision])
cmd.extend([ref])
step_result = self._run(
*cmd,
ok_ret=(0, self.NoSuchProjectError.EXIT_CODE, self.RebaseError.EXIT_CODE),
)
if step_result.retcode == self.NoSuchProjectError.EXIT_CODE:
raise self.NoSuchProjectError(
"Project %s is not part of the checkout" % project
)
elif step_result.retcode == self.RebaseError.EXIT_CODE:
raise self.RebaseError("Failed to rebase")
return step_result
def override(self, project, remote, new_revision="HEAD"):
"""Overrides a given project entry with a new revision.
Args:
project (str): name of the project.
remote (str): URL to the remote repository.
new_revision (str or None): new revision to override the project's current.
"""
cmd = ["override", "-revision", new_revision, project, remote]
return self._run(*cmd)
def resolve(
self,
manifests,
local_manifest=False,
enable_project_lock=False,
enable_package_lock=True,
output=None,
step_test_data=None,
):
"""Generate jiri lockfile in json format for <manifest ...>.
If no manifest provided, jiri will use .jiri_manifest by default.
Args:
manifests (seq of str): manifests to generate a lock file for.
local_manifest (bool): whether to use local manifest.
enable_project_lock (bool): whether to generate locks for projects.
enable_package_lock (bool): whether to generate locks for packages.
output (Path): path to the generated lockfile.
step_test_data (str): test data for the stdout of this command.
"""
cmd = [
"resolve",
"-local-manifest=%r" % local_manifest,
"-enable-project-lock=%r" % enable_project_lock,
"-enable-package-lock=%r" % enable_package_lock,
]
if output:
cmd.extend(["-output", output])
if manifests:
cmd.extend(manifests)
step_result = self._run(
*cmd,
stdout=self.m.raw_io.output_text(),
step_test_data=lambda: self.m.raw_io.test_api.stream_output_text(
step_test_data
),
)
step_result.presentation.logs["output"] = step_result.stdout.split("\n")
return step_result
def snapshot(self, snapshot_file, test_data=None, cipd=False, **kwargs):
"""Generates snapshot that captures the current project state in a manifest.
Args:
snapshot_file (Path): Required output path for the snapshot file.
cipd (bool): Whether to generate additional cipd ensure and version snapshots.
Returns:
A dict mapping file name to snapshot, package ensure and version files.
"snapshot": location of snapshot file.
"ensure": external package ensure file.
"version": external package version file.
"internal_ensure": internal package ensure file.
"internal_version": internal package version file.
"""
cmd = ["snapshot"]
if cipd:
cmd.append("-cipd")
cmd.append(self.m.raw_io.output_text(name="snapshot", leak_to=snapshot_file))
if test_data is None:
test_data = self.test_api.example_snapshot
self._run(
*cmd, step_test_data=lambda: self.test_api.snapshot(test_data), **kwargs
)
snapshot_files = {"snapshot": snapshot_file}
if cipd:
dirname, basename = self.m.path.split(snapshot_file)
snapshot_files["ensure"] = dirname.join(basename + ".ensure")
snapshot_files["version"] = dirname.join(basename + ".version")
snapshot_files["internal_ensure"] = dirname.join(
basename + "_internal.ensure"
)
snapshot_files["internal_version"] = dirname.join(
basename + "_internal.version"
)
return snapshot_files
def snapshot_diff(
self, snapshot_1, snapshot_2, test_data=None, output_file=None, **kwargs
):
"""Generates snapshot diff file between two snapshots.
Args:
snapshot_1 (Path): File or url containing snapshot.
snapshot_2 (Path): File or url containing snapshot.
output_file (Path): Optional output path for the json diff file.
Returns:
Python dict of the snapshot diff between two snapshots in json format.
Max CLs returned for a project is controlled by flag max-cls and is 5
by default.
"""
cmd = [
"diff",
snapshot_1,
snapshot_2,
]
if test_data is None:
test_data = self.test_api.example_snapshot_diff
step = self._run(
*cmd,
stdout=self.m.json.output(name="snapshot diff", leak_to=output_file),
step_test_data=lambda: self.test_api.snapshot_diff(test_data),
**kwargs,
)
return step.stdout
def source_manifest(self, output_file=None, test_data=None, **kwargs):
"""Generates a source manifest JSON file.
Args:
output_file (Path): Optional output path for the source manifest.
Returns:
The contents of the source manifest as a Python dictionary.
"""
cmd = [
"source-manifest",
self.m.json.output(name="source manifest", leak_to=output_file),
]
if test_data is None:
test_data = self.test_api.example_source_manifest
step = self._run(
*cmd,
step_test_data=lambda: self.test_api.source_manifest(test_data),
**kwargs,
)
return step.json.output
def read_manifest_element(
self, manifest, element_type, element_name, name=None, **kwargs
):
"""Reads information about a <project>, <import> or <package> from a manifest
file.
Args:
manifest (str or Path): Path to the manifest file.
element_type (str): One of 'import', 'project' or 'package'.
element_name (str): The name of the element.
name (str): The name of the step.
kwargs (dict): Passed through to __manifest.
Returns:
A dict containing the project fields. Any fields that are missing or have
empty values are omitted from the dict. Examples:
# Read remote attribute of the returned project
print(project['remote']) # https://fuchsia.googlesource.com/my_project
# Check whether remote attribute was present and non-empty.
if 'remote' in project:
...
"""
if element_type == "project":
# This is a Go template matching the schema in pkg/text/template. See docs
# for `__manifest` for more details. We format the template as JSON to
# make it easy to parse into a dict. The template contains the fields in
# a manifest <project>. See //jiri/project/manifest.go for the original
# definition. Add fields to this template as-needed.
template = """
{
"gerrithost": "{{.GerritHost}}",
"githooks": "{{.GitHooks}}",
"historydepth": "{{.HistoryDepth}}",
"name": "{{.Name}}",
"path": "{{.Path}}",
"remote": "{{.Remote}}",
"remotebranch": "{{.RemoteBranch}}",
"revision": "{{.Revision}}"
}
"""
elif element_type == "import":
# This template contains the fields in a manifest <import>. See
# //jiri/project/manifest.go for the original definition.
template = """
{
"manifest": "{{.Manifest}}",
"name": "{{.Name}}",
"remote": "{{.Remote}}",
"revision": "{{.Revision}}",
"remotebranch": "{{.RemoteBranch}}",
"root": "{{.Root}}"
}
"""
else:
assert element_type == "package"
# This template contains the fields in a manifest <package>. See
# //jiri/project/manifest.go for the original definition.
template = """
{
"name": "{{.Name}}",
"version": "{{.Version}}",
"path": "{{.Path}}",
"internal": "{{.Internal}}",
"attributes": "{{.Attributes}}",
"platforms": "{{.Platforms}}"
}
"""
if not name:
name = "read manifest for %s" % element_name
kwargs.setdefault(
"step_test_data", lambda: self.m.json.test_api.output_stream({})
)
element_json = self.__manifest(
manifest=manifest,
element_name=element_name,
template=template,
stdout=self.m.json.output(),
name=name,
**kwargs,
).stdout
# Strip whitespace from any attribute values. Discard empty values.
return {k: v.strip() for k, v in element_json.items() if v.strip()}
def __manifest(self, manifest, element_name, template, **kwargs):
"""Reads information about a supported element (project, import, or package)
from a manifest file.
The template argument is a Go template string matching the schema defined
in pkg/text/template: https://golang.org/pkg/text/template/#hdr-Examples.
Any of the elements's fields may be specified in the template.
See https://fuchsia.googlesource.com/jiri/+/main/project/project.go for a
list of all of the elements' fields.
Example Usage:
# Read the remote= attribute of some <project>.
#
# Example output: https://code.com/my_project.git
__manifest(manifest='manifest', element_name='my_project',
template='{{.Remote}}')
# Read the remote= and path= attributes from some <import>, and
# format the output as "$remote is cloned to $path".
#
# Example output: https://code.com/my_import.git is cloned to /my_import.
__manifest(manifest='manifest', element_name='my_import',
template='{{.Remote}}) is cloned to {{.Path}}')
Args:
manifest (str or Path): Path to the manifest file.
element_name (str): The name of the element to read from.
template (str): A Go template string matching pkg/text/template.
Returns:
The filled-in template string. If the element did not have a value for
some field in the template, the empty string is filled-in for that
field.
"""
return self._run(
"manifest",
"-element",
element_name,
"-template",
template,
manifest,
**kwargs,
)
def generate_gitmodules(
self, gitmodules_path, generate_script=None, redirect_root=False
):
cmd = ["generate-gitmodules"]
if redirect_root:
cmd.append("--redir-root")
if generate_script:
cmd.extend(["--generate-script", generate_script])
cmd.append(gitmodules_path)
return self._run(*cmd)
@property
def runhooks_env(self):
"""Returns env so Jiri hooks can behave differently in infrastructure.
See http://fxb/59015 for why $HOME is overridden.
"""
assert self.m.context.cwd
return {
"HOME": self.m.context.cwd,
"INFRA_RECIPES": "1",
}
def _run(self, *args, **kwargs):
"""Return a jiri command step."""
subcommand = args[0] # E.g., 'init' or 'update'
flags = self._common_flags + list(args[1:])
full_cmd = [self.jiri, subcommand] + flags
name = kwargs.pop("name", "jiri " + subcommand)
return self.m.step(name, full_cmd, **kwargs)