| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import contextlib |
| import functools |
| import re |
| from urllib.parse import urlparse |
| |
| from recipe_engine import recipe_api |
| |
| from . import patch |
| |
| # Set as an output property and consumed by other recipes code and the results |
| # uploader code in google3. |
| # It's a monotonic integer that corresponds to integration revisions so we |
| # can stuff our results into systems that expect Piper changelist numbers. |
| REVISION_COUNT_PROPERTY = "integration-revision-count" |
| |
| # Set as an output property and consumed by the |
| # go/cq-incremental-builder-monitor_dev dashboard. |
| CHECKOUT_FAILED_PROPERTY = "checkout_failed" |
| |
| # Set as an output property and consumed by the |
| # go/cq-incremental-builder-monitor dashboard as well as other incremental |
| # build dashboards. |
| CACHED_REVISION_PROPERTY = "cached_revision" |
| |
| # By default, skip patching GerritChanges which map to these projects. They are |
| # not valid projects to patch into standard Fuchsia checkouts. |
| SKIP_PATCH_PROJECTS = ("infra/recipes",) |
| |
| |
| class _CheckoutResults: |
| """Represents a Fuchsia source checkout.""" |
| |
| def __init__( |
| self, |
| api, |
| root_dir, |
| project, |
| snapshot_file, |
| release_branch, |
| release_version, |
| source_info, |
| clang_toolchain_dir="", |
| gcc_toolchain_dir="", |
| qemu_dir="", |
| rust_toolchain_dir="", |
| cache_enabled=False, |
| ): |
| self._api = api |
| self._root_dir = root_dir |
| self._project = project |
| self._snapshot_file = snapshot_file |
| self._release_branch = release_branch |
| self._release_version = release_version |
| self.source_info = source_info |
| self._clang_toolchain_dir = clang_toolchain_dir |
| self._gcc_toolchain_dir = gcc_toolchain_dir |
| self._qemu_dir = qemu_dir |
| self._rust_toolchain_dir = rust_toolchain_dir |
| self._changed_files_cache = None |
| self._cache_enabled = cache_enabled |
| |
| @property |
| def root_dir(self): |
| """The path to the root directory of the jiri checkout.""" |
| return self._root_dir |
| |
| @property |
| def snapshot_file(self): |
| """The path to the jiri snapshot file.""" |
| return self._snapshot_file |
| |
| @property |
| def release_branch(self): |
| """Release branch corresponding to checkout if applicable, otherwise None.""" |
| return self._release_branch |
| |
| @property |
| def release_version(self): |
| """Release version of checkout if applicable, otherwise None. |
| |
| Returns a release.ReleaseVersion object. |
| """ |
| return self._release_version |
| |
| @property |
| def clang_toolchain_dir(self): |
| return self._clang_toolchain_dir |
| |
| @clang_toolchain_dir.setter |
| def clang_toolchain_dir(self, value): |
| self._clang_toolchain_dir = value |
| |
| @property |
| def gcc_toolchain_dir(self): |
| return self._gcc_toolchain_dir |
| |
| @property |
| def qemu_dir(self): |
| return self._qemu_dir |
| |
| @property |
| def rust_toolchain_dir(self): |
| return self._rust_toolchain_dir |
| |
| def project(self, project_name, **kwargs): |
| return self._api.checkout.project( |
| project_name, checkout_root=self.root_dir, **kwargs |
| ) |
| |
| def project_path(self, project_name, **kwargs): |
| metadata = self.project(project_name, **kwargs) |
| return self._api.path.abs_to_path(metadata["path"]) |
| |
| @property |
| def integration_revision(self): |
| # If triggered by integration CQ, then recipe_wrapper will ensure the |
| # input gitiles_commit is what we want. |
| if self.contains_integration_patch: |
| return self._api.buildbucket.build.input.gitiles_commit.id |
| # Otherwise just use the revision that we actually checked out. |
| return [repo for repo in self.source_info if repo["name"] == self._project][0][ |
| "revision" |
| ] |
| |
| @property |
| def contains_integration_patch(self): |
| """Returns whether we're testing an integration change.""" |
| changes = self._api.buildbucket.build.input.gerrit_changes |
| return ( |
| changes |
| and self._api.checkout.is_integration_project(changes[0].project) |
| and self._api.checkout.is_integration_project(self._project) |
| ) |
| |
| def _upload_source_manifest(self, gcs_bucket, namespace=None): |
| """Upload the jiri source manifest to GCS.""" |
| assert gcs_bucket |
| with self._api.context(cwd=self._root_dir): |
| source_manifest = self._api.jiri.source_manifest() |
| with self._api.step.nest("upload source manifest"): |
| self._api.gsutil.upload_namespaced_file( |
| source=self._api.json.input(source_manifest), |
| bucket=gcs_bucket, |
| subpath="source_manifest.json", |
| namespace=namespace, |
| ) |
| |
| def upload_results(self, gcs_bucket, namespace=None): |
| """Upload snapshot to a given GCS bucket.""" |
| assert gcs_bucket |
| with self._api.step.nest("upload checkout results") as presentation: |
| self._api.gsutil.upload_namespaced_file( |
| source=self.snapshot_file, |
| bucket=gcs_bucket, |
| subpath=self._api.path.basename(self.snapshot_file), |
| namespace=namespace, |
| ) |
| with self._api.context(cwd=self._root_dir.join("integration")): |
| presentation.properties[ |
| REVISION_COUNT_PROPERTY |
| ] = self._api.git.rev_list_count( |
| "HEAD", |
| step_name=f"set {REVISION_COUNT_PROPERTY} property", |
| test_data="1", |
| ) |
| if not self._api.platform.is_mac: |
| self._upload_source_manifest(gcs_bucket, namespace=namespace) |
| |
| def list_files(self, **kwargs): |
| """Returns a list of paths across the checkout. |
| |
| Args: |
| **kwargs (dict): Passed through to `api.git.ls_files()`. |
| |
| Returns: |
| List of Paths. |
| """ |
| all_paths = [] |
| with self._api.step.nest("list files"): |
| projects = self._api.jiri.project( |
| test_data=[ |
| { |
| "name": "project", |
| "path": self._api.path.abspath(self._root_dir), |
| } |
| ], |
| ).json.output |
| for project in projects: |
| project_path = self._api.path.abs_to_path(project["path"]) |
| with self._api.context(cwd=project_path): |
| result = self._api.git.ls_files(step_name=project["name"], **kwargs) |
| files = result.stdout.strip("\n").split("\n") |
| all_paths += [project_path.join(f) for f in files] |
| return all_paths |
| |
| def changed_files(self, test_data=("foo.cc", "bar.cc"), **kwargs): |
| """Returns a list of absolute paths that were changed. |
| |
| Checks the git repo specified in buildbucket_input.gerrit_changes[0]. |
| |
| Args: |
| test_data (seq(str)): Mock list of changed files. |
| **kwargs (dict): Passed through to `api.git.get_changed_files()`. |
| |
| Returns: |
| Empty list if input gerrit_changes is empty or if the build is |
| triggered by a change to a repo that's not included in the |
| checkout. List of Paths otherwise. |
| """ |
| bb_input = self._api.buildbucket.build.input |
| if not bb_input.gerrit_changes: |
| return [] |
| |
| kwargs.setdefault("base", self.release_branch or "origin/main") |
| cache_key = tuple(kwargs.items()) |
| if not self._changed_files_cache or cache_key != self._changed_files_cache[0]: |
| with self._api.step.nest("get changed files"): |
| change = bb_input.gerrit_changes[0] |
| project = change.project |
| with self._api.context(cwd=self._root_dir): |
| project_test_data = [ |
| { |
| "name": project, |
| "path": self._api.path.abspath(self._root_dir) |
| if project == "project" |
| else self._api.path.abspath(self._root_dir.join(project)), |
| } |
| ] |
| try: |
| repo_path = self.project_path( |
| project, test_data=project_test_data |
| ) |
| except self._api.jiri.NoSuchProjectError: |
| return [] |
| with self._api.context(cwd=repo_path): |
| changed_files = self._api.git.get_changed_files( |
| test_data=test_data, **kwargs |
| ) |
| changed_files = [ |
| self._api.path.join(repo_path, changed) for changed in changed_files |
| ] |
| # We only expect this function to be called with one key per build, so |
| # keeping a cache of one element should be sufficient, while still |
| # being correct in case it is called with different keys. |
| self._changed_files_cache = (cache_key, changed_files) |
| return self._changed_files_cache[1] |
| |
| def check_clean(self): |
| # Wrap the check-clean step in an incremental cache guard so the cache |
| # gets cleared if the checkout is dirty. Otherwise dirty checkouts could |
| # persist in the cache and break the clean check on subsequent builds as |
| # well. |
| with self._api.checkout._incremental_cache_ctx( |
| self._cache_enabled |
| ), self._api.context(cwd=self._root_dir): |
| self._api.jiri.check_clean() |
| |
| |
| def _nest(func): |
| """Nest function call within "checkout" step. |
| |
| Check whether already inside a "checkout" step since some public |
| methods in CheckoutApi call other public methods. |
| """ |
| |
| @functools.wraps(func) |
| def wrapper(self, *args, **kwargs): |
| if self._presentation: |
| # Nesting was already set up by a parent function call, no need to |
| # do anything else. |
| return func(self, *args, **kwargs) |
| |
| # The various checkout methods should internally categorize each |
| # internal step correctly as infra or non-infra. Wrapping the entire |
| # call with infra_steps=True would cause each step to be an infra step |
| # by default rather than a non-infra step, leading to steps being |
| # incorrectly marked as infra failures. |
| assert ( |
| not self.m.context.infra_step |
| ), f"api.checkout.{func.__name__}() must not be wrapped with api.context(infra_steps=True)" |
| |
| with self.m.step.nest("checkout") as pres: |
| self._presentation = pres |
| try: |
| ret = func(self, *args, **kwargs) |
| except: |
| pres.properties[CHECKOUT_FAILED_PROPERTY] = True |
| raise |
| self._presentation = None |
| return ret |
| |
| return wrapper |
| |
| |
| class CheckoutApi(recipe_api.RecipeApi): |
| """An abstraction over how Jiri checkouts are created during Fuchsia CI/CQ |
| builds.""" |
| |
| CHECKOUT_INFO_PROPERTY = "checkout_info" |
| GOT_REVISION_PROPERTY = "got_revision" |
| REVISION_COUNT_PROPERTY = REVISION_COUNT_PROPERTY |
| CACHED_REVISION_PROPERTY = CACHED_REVISION_PROPERTY |
| |
| # An invalid patch file represents a user error not an infra |
| # failure. A user should correct their change. |
| class PatchFileValidationError(recipe_api.StepFailure): |
| pass |
| |
| def __init__(self, props, *args, **kwargs): |
| super().__init__(*args, **kwargs) |
| self._presentation = None |
| self._gitiles_commit = None |
| |
| if props.gitiles_commit.host: |
| self._gitiles_commit = props.gitiles_commit |
| self._respect_gitiles_commit_with_gerrit_change = ( |
| props.respect_gitiles_commit_with_gerrit_change |
| ) |
| self._cherry_pick_patches = props.cherry_pick_patches |
| |
| self._clang_toolchain = props.clang_toolchain |
| self._clang_toolchain_dir = None |
| self._gcc_toolchain = props.gcc_toolchain |
| self._gcc_toolchain_dir = None |
| self._qemu = props.qemu |
| self._qemu_dir = None |
| self._rust_toolchain = props.rust_toolchain |
| self._rust_toolchain_dir = None |
| |
| def CheckoutResults(self, *args, **kwargs): |
| """Return a CheckoutResults object. |
| |
| Outside this module, should only be used in testing example recipes. |
| """ |
| return _CheckoutResults(self.m, *args, **kwargs) |
| |
| @_nest |
| def from_spec(self, checkout_spec, **kwargs): |
| """Initialize a Fuchsia checkout according to a checkout spec. |
| |
| Args: |
| checkout_spec (infra.fuchsia.Fuchsia.Checkout): Checkout spec |
| protobuf. |
| **kwargs (dict): Passed through to fuchsia_with_options(). |
| """ |
| return self.fuchsia_with_options( |
| project=checkout_spec.project, |
| manifest=checkout_spec.manifest, |
| remote=checkout_spec.remote, |
| attributes=checkout_spec.attributes, |
| is_release_version=checkout_spec.is_release_version, |
| allow_skipping_patch=checkout_spec.allow_skipping_patch, |
| enable_submodules=checkout_spec.enable_submodules, |
| rebase_patch=not checkout_spec.do_not_rebase_patch, |
| **kwargs, |
| ) |
| |
| @_nest |
| def fuchsia_with_options( |
| self, |
| manifest, |
| remote, |
| path=None, |
| project="integration", |
| is_release_version=False, |
| timeout_secs=45 * 60, |
| use_incremental_cache=False, |
| enable_submodules=False, |
| **kwargs, |
| ): |
| """Uses Jiri to check out a Fuchsia project. |
| |
| The root of the checkout is returned via _CheckoutResults.root_dir. |
| |
| Args: |
| manifest (str): A path to the manifest in the remote (e.g. |
| manifest/minimal). |
| remote (str): A URL to the remote repository which Jiri will be |
| pointed at. |
| project (str): The name that jiri should assign to the project. |
| is_release_version (bool): Whether the checkout is a release |
| version. |
| timeout_secs (int): A timeout to assign to each Jiri operation. |
| use_incremental_cache (bool): Whether to reuse a checkout from the |
| local checkout cache, necessary when the checkout will be used |
| for an incremental build. |
| enable_submodules (bool): Whether to enable submodules. |
| **kwargs (dict): Passed through to with_options(). |
| |
| Returns: |
| A _CheckoutResults containing details of the checkout. |
| """ |
| if path is None: |
| path = self.m.path.start_dir.join("fuchsia") |
| if use_incremental_cache: |
| path = self.m.path.cache_dir.join("incremental") |
| |
| with self._incremental_cache_ctx(use_incremental_cache): |
| self.with_options( |
| manifest=manifest, |
| remote=remote, |
| path=path, |
| project=project, |
| timeout_secs=timeout_secs, |
| use_incremental_cache=use_incremental_cache, |
| enable_submodules=enable_submodules, |
| **kwargs, |
| ) |
| |
| with self.m.context(infra_steps=True, cwd=path): |
| source_info = self.m.jiri.project(name="source-info").json.output |
| |
| snapshot_file = self.m.path.cleanup_dir.join("jiri_snapshot.xml") |
| self.m.jiri.snapshot(snapshot_file) |
| # Always log snapshot contents (even if uploading to GCS) to help debug |
| # things like tryjob failures during roller-commits. |
| self.m.file.read_text("read snapshot", snapshot_file) |
| |
| # If using a release version, resolve release version and release |
| # branches. |
| release_version = release_branch = None |
| if is_release_version: |
| release_version = self._get_release_version(path) |
| release_branch = self._get_release_branch() |
| |
| checkout_result = self.CheckoutResults( |
| root_dir=path, |
| project=project, |
| snapshot_file=snapshot_file, |
| release_branch=release_branch, |
| release_version=release_version, |
| source_info=source_info, |
| clang_toolchain_dir=self._clang_toolchain_dir, |
| gcc_toolchain_dir=self._gcc_toolchain_dir, |
| qemu_dir=self._qemu_dir, |
| rust_toolchain_dir=self._rust_toolchain_dir, |
| cache_enabled=use_incremental_cache, |
| ) |
| |
| # Verify the checkout starts in a clean state. |
| checkout_result.check_clean() |
| |
| return checkout_result |
| |
| @_nest |
| def with_options(self, remote, path=None, **kwargs): |
| """Wrapper to avoid deeply nesting the function body. |
| |
| Only context manager-related logic should go in this function. All |
| other logic should go in _with_options(). |
| |
| Args: see _with_options(). |
| """ |
| if path is None: |
| path = self.m.path.start_dir.join(self.m.git.remote_alias(remote)) |
| self.m.file.ensure_directory("ensure checkout dir", path) |
| |
| with self.m.context(infra_steps=True, cwd=path): |
| return self._with_options(remote=remote, path=path, **kwargs) |
| |
| def _with_options( |
| self, |
| manifest, |
| remote, |
| path=None, |
| project=None, |
| attributes=(), |
| build_input=None, |
| fetch_packages=True, |
| enable_submodules=False, |
| use_lock_file=False, |
| skip_patch_projects=SKIP_PATCH_PROJECTS, |
| timeout_secs=None, |
| use_incremental_cache=False, |
| allow_skipping_patch=False, |
| submodule_update=True, |
| rebase_patch=True, |
| ): |
| """Initializes and populates a jiri checkout from a remote manifest. |
| |
| If a gitiles_commit was provided through the "gitiles_commit" property, |
| this will set the buildbucket.build.input's gitiles_commit to the |
| gitiles_commit from the property. |
| |
| Args: |
| manifest (str): Relative path to the manifest in the remote repository. |
| remote (str): URL to the remote repository. |
| path (Path): The Fuchsia checkout root. If unset, a dedicated |
| checkout directory will be created and returned. |
| project (str): The name that jiri should assign to the project. |
| attributes (seq(str)): A list of jiri manifest attributes; projects or |
| packages with matching attributes - otherwise regarded as optional - |
| will be downloaded. |
| build_input (buildbucket.build_pb2.Build.Input): The input to a buildbucket |
| build. |
| fetch_packages (bool): Whether or not to fetch CIPD packages (and |
| run jiri hooks). Running hooks could theoretically be a separate |
| parameter but in practice there are no use cases for fetching |
| packages without running hooks. And when we want to disable |
| both, we generally care more about disabling fetching packages |
| since it normally takes the longest time. |
| enable_submodules (bool): Whether to enable submodules in jiri config. |
| use_lock_file (bool): Whether to enforce lock files in the jiri |
| root. |
| skip_patch_projects (seq(str)): Do not attempt to patch these |
| projects. |
| timeout_secs (int): A timeout to assign to each Jiri operation. |
| use_incremental_cache (bool): Whether or not this checkout will be |
| used for an incremental build. |
| allow_skipping_patch (bool): Whether to ignore patch failures |
| resulting from the affected project not existing in the |
| checkout. |
| submodule_update (bool): Whether or not to reconcile differences between |
| integration and submodules. |
| rebase_patch (bool): Whether or not to rebase the patch on top of HEAD. |
| |
| Returns: A Path to the checkout root. |
| """ |
| assert manifest, "'manifest' must be set" |
| assert remote, "'remote' must be set" |
| |
| if path == self.m.path.start_dir: |
| # start_dir tends to be a dumping ground, so to keep the checkout |
| # clean it's best to use a dedicated directory. |
| raise ValueError("checkout must not be rooted at start_dir") |
| |
| build_input = build_input or self.m.buildbucket.build.input |
| if self._gitiles_commit: |
| # Override build input gitiles_commit with gitiles_commit from |
| # properties. |
| build_input.gitiles_commit.CopyFrom(self._gitiles_commit) |
| |
| # The revision of the manifest repository to import. We'll do any |
| # patches and overrides on top of the checkout determined by this |
| # version of the manifest repository. |
| base_manifest_revision = None |
| # If the base_manifest_revision is manually provided in the properties |
| # when using led to run the build, use that revision instead of trying |
| # to resolve a new one. An example in which case one might want to set |
| # this property is if the manifest repo differs from the gitiles commit |
| # repo and you want to test at a specific manifest revision. |
| if "base_manifest_revision" in build_input.properties: |
| base_manifest_revision = build_input.properties["base_manifest_revision"] |
| |
| commit = None |
| gerrit_change = None |
| patches = [] # Details of projects to patch (used in CQ). |
| overrides = [] # Details of projects to override (used in local CI). |
| if build_input.gerrit_changes: |
| assert ( |
| len(build_input.gerrit_changes) == 1 |
| ), "build input contains more than one gerrit_change" |
| gerrit_change = build_input.gerrit_changes[0] |
| |
| change_details, current_revision = self._get_change_details(gerrit_change) |
| # Re-resolve HEAD rather than using the base commit resolved by |
| # recipe_wrapper, even for integration patches, because we want to |
| # wait as late as possible to choose a base commit for the checkout. |
| # This will make it more likely that we'll catch merge conflicts and |
| # other types of collisions between changes. |
| # |
| # The most notable risk of doing this is that the properties already |
| # resolved by recipe_wrapper might come from a different version of |
| # the integration repo than the version included in this checkout, |
| # which could cause issues if the properties are not |
| # forwards-compatible with newer versions of the integration repo |
| # (e.g. it references a jiri project that is deleted from the |
| # manifests). However, this is extremely rare in practice because |
| # since we rarely make changes that simultaneously affect infra and |
| # non-infra parts of integration like jiri manifests. |
| # |
| # Only skip this resolution if we are specifically allowing a |
| # Gitiles commit to be combined with a Gerrit change. |
| if not base_manifest_revision and not ( |
| build_input.gitiles_commit.id |
| and self._respect_gitiles_commit_with_gerrit_change |
| ): |
| change_branch = change_details["branch"] |
| if "recipes_integration_ref_mapping" in build_input.properties: |
| ref_mapping = build_input.properties[ |
| "recipes_integration_ref_mapping" |
| ] |
| if f"refs/heads/{change_branch}" in ref_mapping: |
| change_branch = ref_mapping[ |
| f"refs/heads/{change_branch}" |
| ].replace("refs/heads/", "") |
| base_manifest_revision = self._resolve_branch_head( |
| remote, branch=change_branch |
| ) |
| |
| if gerrit_change.project not in skip_patch_projects: |
| patches.append( |
| { |
| "host": f"https://{gerrit_change.host}", |
| "project": gerrit_change.project, |
| "ref": current_revision["ref"], |
| } |
| ) |
| # TODO(olivernewman): Also load patches.json into the "patches" list |
| # (which must happen after patching in the gerrit change so we have |
| # patches.json locally) so we can expose all patches in the checkout |
| # info output property. |
| |
| # Only use the Gitiles commit to resolve the base manifest revision if |
| # it wasn't already resolved in the previous section. |
| if build_input.gitiles_commit.id: |
| commit = build_input.gitiles_commit |
| manifest_remote_url = urlparse(remote) |
| host = commit.host |
| # When using sso we only specify the lowest subdomain, by convention. |
| if manifest_remote_url.scheme == "sso": |
| host = host.split(".")[0] |
| commit_remote = f"{manifest_remote_url.scheme}://{host}/{commit.project}" |
| is_manifest_commit = commit_remote == remote |
| if is_manifest_commit: |
| base_manifest_revision = commit.id |
| else: |
| if not base_manifest_revision: |
| base_manifest_revision = self._resolve_branch_head(remote, "main") |
| # In order to identify a project to override, jiri keys on both |
| # the project name and the remote source repository (not to be |
| # confused with `remote`, the manifest repository). Doing this |
| # correctly would require finding the commit's remote in the |
| # transitive imports of the jiri manifest. But those transitive |
| # imports aren't available until we run "jiri update", and doing |
| # that twice is slow, so we rely on: |
| # 1. The convention that the name of the jiri project |
| # is the same as commit.project. |
| # 2. The hope that the URL scheme of the commit remote is the |
| # same as that of the manifest remote. |
| overrides.append( |
| { |
| "project": commit.project, # See 1. above |
| "remote": commit_remote, |
| "new_revision": commit.id, |
| } |
| ) |
| if not base_manifest_revision: |
| # If we haven't resolved the base manifest revision by this point, |
| # we have neither a triggering commit nor a triggering Gerrit |
| # change, so just checkout the manifest repository at HEAD. |
| base_manifest_revision = self._resolve_branch_head(remote, "main") |
| |
| cached_revision = "" |
| if use_incremental_cache: |
| resolved_revision = self._resolve_cached_revision(project) |
| if resolved_revision: |
| cached_revision = resolved_revision |
| |
| # TODO(https://fxbug.dev/319943920): Reenable on integration patches once bug is fixed. |
| has_integration_patch = False |
| for p in patches: |
| if self.is_integration_project(p["project"]): |
| has_integration_patch = True |
| break |
| enable_submodules = False if has_integration_patch else enable_submodules |
| self.m.jiri.init( |
| directory=path, |
| dissociate=use_incremental_cache, |
| attributes=attributes, |
| enable_submodules=enable_submodules, |
| use_lock_file=use_lock_file, |
| ) |
| |
| self.m.file.read_text("read jiri config", path.join(".jiri_root", "config")) |
| |
| self.m.jiri.import_manifest( |
| manifest, |
| remote, |
| name=project, |
| revision=base_manifest_revision, |
| overwrite=use_incremental_cache, |
| ) |
| |
| for override in overrides: |
| self.m.jiri.override(**override) |
| |
| # Resets the checkout to make sure that nothing remains in the cache |
| # from a previous build. |
| if use_incremental_cache: |
| with self.m.context(cwd=path): |
| self.m.jiri.reset() |
| |
| # We must clone all projects prior to applying any patches. But we need |
| # not run hooks or fetch packages until after all patches are applied, |
| # since applying a patch to the manifest repository could update the |
| # versions of the packages we need to fetch. |
| self.m.jiri.update( |
| run_hooks=False, |
| fetch_packages=False, |
| timeout=timeout_secs, |
| gc=True, |
| ) |
| |
| successful_patches = [] |
| for p in patches: |
| is_manifest_patch = project == p["project"] |
| # Failures in pulling down patches and rebasing are likely not |
| # infra-related. If we got here, we're already able to talk to Gerrit |
| # successfully, so any errors are likely merge conflicts. |
| with self.m.context(infra_steps=False): |
| try: |
| patch_base_revision = self._apply_patch( |
| path, |
| p["ref"], |
| # TODO(https://fxbug.dev/328639365): This should point |
| # to smart-integration once jiri knows how to apply an |
| # integration patch to a smart-integration project. |
| p["project"], |
| gerrit_change, |
| change_details, |
| is_manifest_patch, |
| timeout_secs, |
| enable_submodules, |
| submodule_update, |
| rebase=rebase_patch, |
| ) |
| except self.m.jiri.NoSuchProjectError as e: |
| # Always allow skipping integration patches to support |
| # running tryjobs on infra config changes even when the |
| # tryjob's checkout doesn't include the integration repo. |
| if allow_skipping_patch or self.is_integration_project( |
| p["project"] |
| ): |
| self.m.step.empty("skipping patch", step_text=str(e)) |
| continue |
| # Aside from the integration repo, there isn't a reason to |
| # run tryjobs on repos that aren't included in the checkout. |
| # So we should fail the build to notify the user that the |
| # tryjob will not actually test their change. |
| raise |
| # Emit the revision that we rebased the patch change on top of, |
| # which will always be head of the patch's target branch at this |
| # moment in time. |
| p["base_revision"] = patch_base_revision |
| # If patch is updating integration manifest, we need to make submodules |
| # are updated. |
| successful_patches.append(p) |
| |
| # We should only emit metadata for projects that we successfully |
| # patched. |
| patches = successful_patches |
| |
| # When submodules are enabled, we should make sure integration pins for |
| # submodules are updated as source of truth for projects are now in |
| # submodules. |
| if enable_submodules and submodule_update: |
| self._submodule_update(path, base_manifest_revision) |
| self.m.git.status() |
| if self._repo_has_uncommitted_files(path, check_untracked=False): |
| self.m.git.add(only_tracked=True) |
| self.m.git.commit( |
| message="Update submodule git links", |
| all_tracked=True, |
| allow_empty=True, |
| ) |
| self.m.git.submodule_status() |
| |
| if use_incremental_cache: |
| # If the cache is warm, the previous commit that was tested on the |
| # same bot could have caused an untracked file to be written to the |
| # checkout via a git hook, new CIPD package, or build system change. |
| # If the commit also added a .gitignore entry for the untracked |
| # file, the file may still exist in the cache but no longer be |
| # git-ignored. So remove all files that are untracked and not |
| # git-ignored (also removing git-ignored files would force |
| # unnecessary redownloading of CIPD packages). |
| # |
| # For simplicity and determinism, do the clean unconditionally even |
| # if the cache is cold. |
| # |
| # It's important to do this *before* running hooks and fetching |
| # packages so that the subsequent check-clean step catches files |
| # added by git hooks or jiri packages that are not properly |
| # git-ignored. |
| self.m.jiri.clean() |
| |
| # Run hooks and fetch CIPD packages separately from `jiri update` to get |
| # timing information. We want to fetch packages only *after* any gerrit |
| # changes have been patched in. If we fetched packages *before* patching |
| # in a change to the manifest repository, then we'd need to run hooks |
| # again afterward to honor the contents of the patch, and might end up |
| # overwriting old versions of CIPD packages that were downloaded prior |
| # to patching. So it's more efficient to only fetch packages once at the |
| # very end. |
| if fetch_packages: |
| # Handle package_overrides.json, if present. |
| packages_to_skip = self._apply_package_overrides(path) |
| self.m.jiri.run_hooks( |
| # We must use the local manifest from integration.git rather |
| # than the contents of .jiri_manifest in cases where we might |
| # have patched in a change to the manifest repository. The only |
| # time we *cannot* use the local manifest is when there are |
| # overrides, in which case .jiri_manifest will be the only |
| # correct source of truth. |
| local_manifest=bool(gerrit_change), |
| fetch_packages=True, |
| # Jiri sets the fetch-packages timeout to 5x the hook timeout. |
| hook_timeout_secs=timeout_secs / 5 if timeout_secs else None, |
| # Skip downloading any packages that were already replaced by |
| # package_overrides.json. |
| packages_to_skip=packages_to_skip, |
| ) |
| |
| # Fetch custom toolchain if requested. |
| if not self._clang_toolchain_dir: |
| self._clang_toolchain_dir = self._fetch_toolchain( |
| path, "clang", self._clang_toolchain, "third_party/clang" |
| ) |
| if not self._gcc_toolchain_dir: |
| self._gcc_toolchain_dir = self._fetch_toolchain( |
| path, "gcc", self._gcc_toolchain, "third_party/gcc" |
| ) |
| if not self._qemu_dir: |
| self._qemu_dir = self._fetch_toolchain( |
| path, "qemu", self._qemu, "third_party/qemu" |
| ) |
| if not self._rust_toolchain_dir: |
| self._rust_toolchain_dir = self._fetch_toolchain( |
| path, "rust", self._rust_toolchain, "third_party/rust" |
| ) |
| |
| # This information is consumed by `fx sync-to` to reproduce infra |
| # checkouts locally. |
| # TODO(olivernewman): Also expose overrides, and patches specified by |
| # patches.json. Neither overrides (used only in local CI) nor |
| # patches.json (used only in occasional manual experimental builds) is |
| # used very frequently, but they're necessary to include here for |
| # complete correctness. |
| self._presentation.properties[self.CHECKOUT_INFO_PROPERTY] = { |
| "enable_submodules": enable_submodules, |
| "manifest_project": project, |
| "manifest_remote": remote, |
| "manifest": manifest, |
| "base_manifest_revision": base_manifest_revision, |
| "patches": patches, |
| } |
| |
| # This information is consumed by a variety of incremental builder |
| # dashboards. |
| self._presentation.properties[self.CACHED_REVISION_PROPERTY] = cached_revision |
| return path |
| |
| def is_integration_project(self, project): |
| return project in ("integration", "smart-integration") |
| |
| def _repo_has_uncommitted_files(self, repo_dir, check_untracked): |
| """Checks whether the git repository at repo_dir has any changes. |
| |
| Args: |
| repo_dir (Path): Path to the git repository. |
| check_untracked (bool): Whether to include untracked files in the check. |
| |
| Returns: |
| True if there are, and False if not. |
| """ |
| with self.m.context(cwd=repo_dir): |
| step_result = self.m.git.ls_files( |
| step_name="check for no-op commit", |
| modified=True, |
| deleted=True, |
| exclude_standard=True, |
| others=check_untracked, |
| ) |
| step_result.presentation.logs["stdout"] = step_result.stdout.split("\n") |
| return bool(step_result.stdout.strip()) |
| |
| # Update submodule projects pin based on the integration revisoins |
| def _submodule_update(self, path, revision, jiri_projects_path=None): |
| if not jiri_projects_path: |
| jiri_projects_path = self.m.path.mkstemp("jiri-projects-json") |
| self.m.jiri.project(out=jiri_projects_path) |
| snapshot_files = self.m.jiri.snapshot( |
| self.m.path.cleanup_dir.join("cipd"), cipd=True |
| ) |
| # Submodule_update requires commit information for updating CLs. |
| # Currently, we are using tool to make sure submodules are up to date |
| # in case of lag with submodule_update builder. Putting "foo" as a place |
| # holder. Eventually this will be removed. |
| inputs = { |
| "superproject_path": path, |
| "jiri_projects_path": jiri_projects_path, |
| "snapshot_paths": snapshot_files, |
| "current_revision": revision, |
| "host": "foo", |
| "project": "foo", |
| "branch": "foo", |
| "commit_author_name": "foo", |
| "commit_author_email": "foo", |
| "commit_message": "foo", |
| } |
| # Create changes to update subdmoules based on integration |
| return self.m.submodule_update.integration_update( |
| "integration update", inputs, no_commit=True |
| ) |
| |
| def project(self, project_name, checkout_root=None, **kwargs): |
| """Returns metadata for a project in the checkout. |
| |
| Raises NoSuchProjectError if the project is not among the repos in |
| the checkout. |
| """ |
| if not checkout_root: |
| checkout_root = self.m.context.cwd |
| with self.m.context(cwd=checkout_root, infra_steps=True): |
| output = self.m.jiri.project(projects=[project_name], **kwargs).json.output |
| if not output: |
| raise self.m.jiri.NoSuchProjectError( |
| f"project {project_name!r} is not present in the checkout" |
| ) |
| return output[0] |
| |
| def _resolve_cached_revision(self, project): |
| """Return the revision of the project in the cache. |
| |
| Args: |
| project (str): The name of the project to look in the cache for. |
| |
| Returns: |
| A containing a git revision, or None if there is no cached checkout. |
| """ |
| return self.m.jiri.get_import_revision( |
| project, |
| name="get cached revision", |
| test_data="foobar", |
| # The command will fail if a jiri root does not exist (i.e. if |
| # we haven't started a checkout out), which is fine - we'll just |
| # return None. |
| ok_ret="any", |
| ) |
| |
| def _resolve_branch_head(self, remote, branch): |
| """Return the hash of the commit currently at the tip of a branch. |
| |
| Args: |
| remote (str): The URL of the repository containing the branch. |
| branch (str): Name of the branch to resolve (e.g. "main"). |
| """ |
| return self.m.git.get_remote_branch_head( |
| url=self.m.sso.sso_to_https(remote), |
| branch=branch, |
| step_name=f"resolve head of {branch!r} branch", |
| ) |
| |
| def _apply_patch( |
| self, |
| path, |
| patch_ref, |
| patch_project, |
| gerrit_change, |
| change_details, |
| is_manifest_patch, |
| timeout_secs, |
| enable_submodules, |
| submodule_update, |
| rebase, |
| ): |
| target_branch = change_details["branch"] |
| |
| change_remote, change_ref = self.m.gerrit.resolve_change(gerrit_change) |
| try: |
| project_dir = self.m.path.abs_to_path(self.project(patch_project)["path"]) |
| except self.m.jiri.NoSuchProjectError: |
| submodule_dict = self.m.git.list_submodules() |
| submodule_path = None |
| for _, info in submodule_dict.items(): |
| if info.get("url") == change_remote: |
| submodule_path = info.get("path") |
| break |
| if not submodule_path: |
| raise |
| project_dir = path.join(submodule_path) |
| |
| if self._cherry_pick_patches: |
| with self.m.context(cwd=project_dir): |
| self.m.git.fetch(change_remote, change_ref) |
| self.m.git.cherry_pick("FETCH_HEAD") |
| # The repo would normally be in a detached head state after the |
| # cherry-pick, which would cause a subsequent `jiri update` (if |
| # `is_manifest_patch`) to throw away the cherry-pick. To prevent |
| # that, switch to a branch. |
| self.m.git.raw_checkout( |
| branch=f"change/{gerrit_change.change}/{gerrit_change.patchset}" |
| ) |
| else: |
| try: |
| self.m.jiri.patch( |
| patch_ref, |
| host=f"https://{gerrit_change.host}", |
| project=patch_project, |
| rebase=rebase, |
| rebase_branch=target_branch, |
| ) |
| except self.m.jiri.NoSuchProjectError: |
| with self.m.context(cwd=project_dir): |
| self.m.git.fetch(change_remote, change_ref) |
| self.m.git.raw_checkout(ref="FETCH_HEAD") |
| self.m.git.rebase(ref=f"remotes/origin/{target_branch}") |
| |
| with self.m.context(cwd=project_dir): |
| # Resolve the revision that we patched on top of, which is the tip |
| # of the target branch at this moment in time. |
| patch_base_revision = self.m.git.rev_parse( |
| f"origin/{target_branch}", |
| step_test_data=lambda: self.m.raw_io.test_api.stream_output_text( |
| "abc123" |
| ), |
| ) |
| if enable_submodules and submodule_update: |
| # Find projects to check which projects are enabled as submodules. |
| # If we are patching project in integration, we need to make sure jiri projects are |
| # updated based on the new manifest. |
| jiri_projects_path = self.m.path.mkstemp("jiri-projects-json") |
| local_manifest = self.is_integration_project(patch_project) |
| project_info = self.m.jiri.project( |
| name="jiri project submodule check patch", |
| local_manifest=local_manifest, |
| out=jiri_projects_path, |
| ).json.output |
| |
| # TODO(https://fxbug.dev/319943920): Remove `nocover` once |
| # submodules are re-enabled on integration patches. |
| if self.is_integration_project(patch_project): # pragma: nocover |
| self._submodule_update( |
| path, patch_base_revision, jiri_projects_path=jiri_projects_path |
| ) |
| self.m.step.empty( |
| "print patch base revision", |
| log_text=self.m.json.dumps(patch_base_revision, indent=2), |
| ) |
| self.m.git.submodule_status() |
| self.m.git.status() |
| if self._repo_has_uncommitted_files(path, check_untracked=False): |
| self.m.git.add(only_tracked=True) |
| self.m.git.commit( |
| message="Update submodule git links", |
| all_tracked=True, |
| allow_empty=True, |
| ) |
| else: |
| for project in project_info: |
| if project["name"] == patch_project and project.get( |
| "gitsubmoduleof" |
| ): |
| self.m.git.commit( |
| message=f"Update submodules to {gerrit_change.host}/{patch_project}/{patch_ref}, ", |
| all_tracked=True, |
| ) |
| break |
| |
| # Handle patches.json, if present. |
| self._apply_patchfile(path, gerrit_change, enable_submodules) |
| |
| if is_manifest_patch: |
| self.m.jiri.update( |
| gc=True, |
| rebase_tracked=True, |
| rebase_submodules=True, |
| local_manifest=True, |
| run_hooks=False, |
| fetch_packages=False, |
| timeout=timeout_secs, |
| ) |
| |
| # It's difficult to figure out what commit the tryjob rebased a CL on |
| # top of. So we simply log the last few commits here. (It's not |
| # sufficient to log just the parent commit, because checking out a CL |
| # at the top of a stack of open CLs will also check out and rebase all |
| # the parent CLs on top of main). |
| with self.m.context(cwd=project_dir): |
| self.m.git.log(depth=10) |
| return patch_base_revision |
| |
| def _apply_patchfile(self, path, gerrit_change, enable_submodules): |
| """Parses and applies the PatchFile for the given gerrit change.""" |
| # TODO: This is a fragile assumption that relies on integration.git |
| # being checked out at //integration. Find a better way to derive path |
| # to patches.json. |
| patchfile_path = path.join(gerrit_change.project, "patches.json") |
| try: |
| # Note that in recipe unit testing mode, `read_json` returns None |
| # string if the file has not been mocked, rather than raising an |
| # exception. |
| contents = self.m.file.read_json("read patches.json", patchfile_path) |
| except self.m.file.Error as e: |
| if e.errno_name != "ENOENT": # pragma: no cover |
| raise |
| self.m.step.active_result.presentation.status = self.m.step.SUCCESS |
| contents = None |
| |
| # The change doesn't include a patch file, so no need to do any |
| # patching. |
| if not contents: |
| self.m.step.active_result.presentation.step_text = "no patch file found" |
| return |
| |
| patch_file = patch.PatchFile.from_json(contents) |
| |
| # Ensure patchfile is valid. |
| validation_err = patch_file.validate(gerrit_change) |
| if validation_err is not None: |
| raise self.PatchFileValidationError(str(validation_err)) |
| |
| # Find projects to check which projects are enabled as submodules. |
| project_info = self.m.jiri.project( |
| name="jiri project submodule check patchfile" |
| ).json.output |
| |
| for patch_input in patch_file.inputs: |
| # If the patch pulls in a project that's not in the workspace already, the patch |
| # would not affect this build / test run. Skip this patch. |
| try: |
| self.project( |
| patch_input.project, name=f"jiri project {patch_input.project}" |
| ) |
| except self.m.jiri.NoSuchProjectError: |
| warning = f"warning: skipping patch for {patch_input.project} which is not in the checkout" |
| self.m.step.empty(warning) |
| continue |
| |
| # Strip protocol if present. |
| host = patch_input.host |
| host_url = urlparse(host) |
| if host_url.scheme: |
| host = host_url.hostname |
| |
| # We need to know the branch that the change is targeting in order |
| # to correctly patch it on top of the destination branch. The branch |
| # is not included in the patchfile so must be obtained via the |
| # Gerrit API. |
| match = re.match( |
| r"refs/changes/\d+/(?P<change_num>\d+)/\d+", |
| patch_input.ref, |
| ) |
| assert match, f"invalid change ref {patch_input.ref}" |
| change_num = match.group("change_num") |
| target_branch = self.m.gerrit.change_details( |
| f"get details for https://{patch_input.host}/{change_num}", |
| host=patch_input.host, |
| change_id=f"{patch_input.project}~{change_num}", |
| test_data=self.m.json.test_api.output( |
| {"branch": f"{patch_input.project}-branch"} |
| ), |
| ).json.output["branch"] |
| |
| # Patch in the change |
| self.m.jiri.patch( |
| ref=patch_input.ref, |
| host=f"https://{host}", |
| project=patch_input.project, |
| rebase=True, |
| rebase_branch=target_branch, |
| ) |
| |
| # When submodules are enabled, fuchsia.git will reset submodules' pins in the next jiri update. |
| # So we need to commit the patch changes here. |
| if not enable_submodules: |
| continue |
| |
| # TODO(https://fxbug.dev/319943920): Remove `nocover` once |
| # submodules are re-enabled on integration patches. |
| for project in project_info: # pragma: nocover |
| if project["name"] == patch_input.project and project.get( |
| "gitsubmoduleof" |
| ): |
| self.m.git.commit( |
| message=f"Update submodules to {host}/{patch_input.project}/{patch_input.ref}, ", |
| all_tracked=True, |
| ) |
| break |
| |
| def _apply_package_overrides(self, path): |
| """Apply one or more package overrides if package_overrides.json is |
| present. package_overrides.json is a map of package to CAS digest pairs. |
| |
| Args: |
| path (Path): Checkout root. |
| |
| Returns: |
| list(str): List of packages in package_overrides.json. |
| """ |
| package_overrides_path = path.join("package_overrides.json") |
| self.m.path.mock_add_paths(package_overrides_path) |
| if not self.m.path.exists(package_overrides_path): # pragma: no cover |
| return [] |
| package_overrides = ( |
| self.m.file.read_json( |
| "read package overrides", |
| package_overrides_path, |
| ) |
| or {} |
| ) |
| for package, cas_digest in package_overrides.items(): |
| package_entries = self.m.jiri.package([package]) |
| # The package may appear more than once across the checkout. |
| for package_entry in package_entries: |
| # Download the CAS tree at the path that the package would be |
| # downloaded to. |
| self.m.cas_util.download( |
| cas_digest, |
| package_entry["path"], |
| ) |
| return package_overrides.keys() |
| |
| def _get_change_details(self, gerrit_change): |
| """Fetches the details of a Gerrit change. |
| |
| Retries if the response does not include details for the desired |
| patchset, as replication lag may cause recently created patchsets to not |
| immediately appear in responses from all Gerrit backends. |
| |
| Returns: |
| A dict representing the change details, and a dict representing the |
| details of the current revision. |
| """ |
| |
| def impl(): |
| step = self.m.gerrit.change_details( |
| name="get change details", |
| change_id=f"{gerrit_change.project}~{gerrit_change.change}", |
| host=gerrit_change.host, |
| query_params=["ALL_REVISIONS"], |
| test_data=self.m.json.test_api.output( |
| { |
| "branch": "main", |
| "revisions": { |
| "d4e5f6": {"_number": 3, "ref": "refs/changes/00/100/3"}, |
| "a1b2c3": {"_number": 7, "ref": "refs/changes/00/100/7"}, |
| "g7h8i9": {"_number": 9, "ref": "refs/changes/00/100/9"}, |
| }, |
| } |
| ), |
| ) |
| current_patchsets = [ |
| rev |
| for rev in step.json.output["revisions"].values() |
| if rev["_number"] == gerrit_change.patchset |
| ] |
| if not current_patchsets: |
| step.presentation.status = self.m.step.INFRA_FAILURE |
| step.presentation.step_text = ( |
| f"missing details for patchset {gerrit_change.patchset}" |
| ) |
| raise self.m.step.InfraFailure( |
| f"Failed to fetch details for patchset {gerrit_change.patchset}." |
| f"This may be due to Gerrit replication lag." |
| ) |
| return step.json.output, current_patchsets[0] |
| |
| return self.m.utils.retry(impl, max_attempts=7, sleep=15, backoff_factor=2) |
| |
| def _get_release_version(self, path): |
| """Get release version corresponding to HEAD.""" |
| with self.m.step.nest("resolve release version") as presentation: |
| release_version = self.m.release.ref_to_release_version( |
| ref="HEAD", |
| repo_path=path.join("integration"), |
| ) |
| # Fuchsia's buildmon service depends on this property being set. |
| presentation.properties["release_version"] = str(release_version) |
| return release_version |
| |
| def _get_release_branch(self): |
| with self.m.step.nest("resolve release branch") as presentation: |
| ref = self.m.buildbucket.build.input.gitiles_commit.ref.replace( |
| "refs/heads/", "" |
| ) |
| if not self.m.release.validate_branch(ref): |
| return None |
| presentation.properties["release_branch"] = ref |
| return ref |
| |
| def _fetch_toolchain(self, path, name, toolchain, cipd_package): |
| """Fetch a prebuilt toolchain from CAS or CIPD. |
| |
| Args: |
| path (Path): The Fuchsia checkout root. |
| name (str): Name of the toolchain (e.g. "clang"). |
| toolchain (CustomToolchain): Information about where to |
| download the toolchain from. |
| cipd_package (str): The name of the CIPD package to download if |
| toolchain_info["type"] is "cipd". |
| |
| Returns: A Path to the root of a directory where the toolchain was downloaded. |
| """ |
| if not (toolchain.cas_digest or toolchain.cipd_version): |
| return "" |
| with self.m.step.nest(f"{name}_toolchain"), self.m.context(infra_steps=True): |
| toolchain_dir = path.join("prebuilt", "third_party", name, "custom") |
| if toolchain.cas_digest: |
| self.m.cas_util.download( |
| step_name="download", |
| digest=toolchain.cas_digest, |
| output_dir=toolchain_dir, |
| cas_instance=toolchain.cas_instance, |
| ) |
| elif toolchain.cipd_version: |
| pkgs = self.m.cipd.EnsureFile() |
| # NOTE: This isn't correct for Rust, which uses separate |
| # host/target CIPD packages, but the Rust package owners don't |
| # care about testing custom CIPD packages. |
| pkgs.add_package( |
| "fuchsia/%s/${platform}" % cipd_package, toolchain.cipd_version |
| ) |
| self.m.cipd.ensure(toolchain_dir, pkgs) |
| return toolchain_dir |
| |
| def _incremental_cache_ctx(self, use_incremental_cache): |
| """Returns a context manager for guarding the incremental checkout dir. |
| |
| ... or else a no-op context manager if incremental caches are disabled. |
| |
| Checkout caches are necessary for supporting incremental builds because |
| doing a fresh checkout would reset all files' mtimes, which would result |
| in a full build because it would appear to the build system that all |
| files had been modified since the last build. |
| |
| Note that incremental builds are specific to the Fuchsia project and are |
| not relevant in the general case of checking out a Jiri project. |
| """ |
| if use_incremental_cache: |
| return self.m.cache.guard("incremental") |
| return contextlib.nullcontext() |
