| # Copyright 2021 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # TODO: fxbug.dev/332707272 - Use dict[], list[], |, etc. instead of these. |
| from typing import ( |
| Dict, |
| List, |
| Optional, |
| Sequence, |
| Tuple, |
| Union, |
| ) |
| |
| from recipe_engine import config_types, recipe_api |
| |
| from PB.recipe_modules.fuchsia.cipd_util.upload_manifest import CIPDUploadManifest |
| from RECIPE_MODULES.recipe_engine.file.api import SymlinkTree |
| from RECIPE_MODULES.recipe_engine.cipd.api import InstallMode, Metadata |
| |
| |
| class HardlinkTree(SymlinkTree): |
| """A representation of a tree of hardlinked files. |
| |
| This tree differs from the SymlinkTree in the resource passed to it. |
| See CIPDUtilApi.hardlink_tree for the public constructor. |
| """ |
| |
| |
| class CIPDUtilApi(recipe_api.RecipeApi): |
| """Utilities for interacting with CIPD.""" |
| |
| @property |
| def platform_name(self) -> str: |
| """Returns CIPD's name for the current platform. |
| |
| This is the value that the CIPD CLI will substitute for ${platform} |
| in any package names that it sees. |
| """ |
| os = self.m.platform.name.replace("win", "windows") |
| arch = {"intel": {32: "386", 64: "amd64"}, "arm": {32: "armv6", 64: "arm64"}}[ |
| self.m.platform.arch |
| ][self.m.platform.bits] |
| return f"{os}-{arch}" |
| |
| def upload_package( |
| self, |
| pkg_name: str, |
| pkg_root: config_types.Path, |
| search_tag: Dict[str, str], |
| pkg_paths: List[config_types.Path] | None = None, |
| repository: Optional[str] = None, |
| install_mode: Optional[InstallMode] = "copy", |
| refs: Sequence[str] = ("latest",), |
| metadata: Optional[List[Union[Tuple[str, str], Metadata]]] = None, |
| add_version_file: bool = True, |
| name: Optional[str] = None, |
| step_name: str = "cipd", |
| verification_timeout: Optional[str] = None, |
| fail_if_exists: bool = False, |
| ) -> str: |
| """Creates and uploads a CIPD package containing the tool at pkg_dir. |
| |
| The tool is published to CIPD under the path pkg_name. |
| |
| Args: |
| pkg_name: The CIPD package to publish to. |
| pkg_root: The absolute path to the parent directory of the package. |
| search_tag: The tag to search for the CIPD pin with. This should contain one |
| element and be either `git_revision` or `version`. |
| pkg_paths: A list of Path objects which specify the paths to directories |
| or files to upload. Defaults to [pkg_root]. |
| repository: The git repository where code for the package lives. |
| install_mode: The install mode for the package. |
| refs: Refs to set on the package. |
| metadata: Metadata to add to the package. |
| add_version_file (bool): Include a .cipd_version file in the package. |
| name: Logical name of the package. Defaults to second-to-last |
| part of pkg_name. |
| step_name: Name of the step. |
| verification_timeout: Passed through to self.m.cipd.register(). |
| fail_if_exists: Fails the build if a package already exists in CIPD |
| with this tag. |
| |
| Returns: |
| The CIPDApi.Pin instance_id. |
| """ |
| pkg_paths = pkg_paths or [pkg_root] |
| |
| with self.m.step.nest(step_name) as presentation: |
| dry_run = self.m.buildbucket_util.is_dev_or_try or not pkg_name |
| if dry_run: |
| presentation.step_text = "dry run" |
| pkg_name = pkg_name or "experimental/fuchsia/cipd-util-test/${platform}" |
| |
| pkg_def = self.m.cipd.PackageDefinition( |
| package_name=pkg_name, package_root=pkg_root, install_mode=install_mode |
| ) |
| |
| # Mock the existence of the package root directory so `isdir()` will |
| # return True. |
| self.m.path.mock_add_directory(pkg_root) |
| for path in pkg_paths: |
| if self.m.path.isdir(path): |
| pkg_def.add_dir(path) |
| else: |
| pkg_def.add_file(path) |
| |
| if not name: |
| # E.g., "fuchsia/go/linux-amd64" -> "go". |
| name = str(pkg_name.split("/")[-2]) |
| if add_version_file: |
| pkg_def.add_version_file(f".versions/{name}.cipd_version") |
| |
| pkg_path = self.m.path.mkdtemp("cipd-util-build").join(f"{name}.pkg") |
| cipd_pin = self.m.cipd.build_from_pkg(pkg_def, pkg_path) |
| |
| # `deferred_steps_dir` is a properties injected by `recipe_wrapper` |
| # and represents a directory that exists on disk if the property is |
| # set. This information is consumed by `recipe_wrapper` to generate |
| # attestations for the CIPD package and will be attached as |
| # additional metadata by the `recipe_wrapper`. |
| deferred_steps_dir = self.m.recipe_wrapper.deferred_steps_dir |
| if deferred_steps_dir: |
| self.m.file.write_json( |
| name=f"write {cipd_pin.package} info to deferred_steps", |
| dest=self.m.path.abs_to_path(deferred_steps_dir).join( |
| f"{cipd_pin.instance_id}.json" |
| ), |
| data={ |
| # This path should be unique, if it is overwritten |
| # during the recipe flow, we may fail to generate |
| # the attestation correctly. |
| # The consumer can trivially verify the sha256 |
| # hash of the file matches the |
| # `cipd_instance_id[:-1]` |
| "path": self.m.path.abspath(pkg_path), |
| "cipd_package": cipd_pin.package, |
| "cipd_instance_id": cipd_pin.instance_id, |
| }, |
| ) |
| |
| if dry_run: |
| return cipd_pin.instance_id |
| |
| assert ( |
| len(search_tag) == 1 |
| ), "search_tag must contain one (key: value) pair to search for." |
| search_tag_key = list(search_tag.keys())[0] |
| search_tag_value = search_tag[search_tag_key] |
| assert ( |
| search_tag_value |
| ), f"value for search tag {search_tag_key!r} is unset: {search_tag_value!r}" |
| cipd_pins = self.m.cipd.search( |
| pkg_name, |
| f"{search_tag_key}:{search_tag_value}", |
| test_instances=[], |
| ) |
| if cipd_pins: |
| if fail_if_exists: |
| raise self.m.step.StepFailure( |
| "Package already exists %s: %s" |
| % (search_tag_key, search_tag_value) |
| ) |
| self.m.step.empty("package is up-to-date") |
| assert len(cipd_pins) == 1, f"{pkg_name} has too many pins" |
| return cipd_pins[0].instance_id |
| |
| tags = {} |
| tags.update(search_tag) |
| |
| final_metadata = [ |
| self.m.cipd.Metadata("bbid", self.m.buildbucket_util.id), |
| ] |
| if repository: |
| final_metadata.append( |
| self.m.cipd.Metadata("git_repository", repository) |
| ) |
| |
| if metadata: |
| for metadatum in metadata: |
| if isinstance(metadatum, self.m.cipd.Metadata): |
| final_metadata.append(metadatum) |
| else: |
| key, value = metadatum |
| final_metadata.append(self.m.cipd.Metadata(key, value)) |
| # TODO(fxbug.dev/85982) Remove metadata from tags. |
| tags[key] = value |
| |
| cipd_pin = self.m.cipd.register( |
| package_name=pkg_name, |
| package_path=pkg_path, |
| refs=refs, |
| tags=tags, |
| metadata=final_metadata, |
| verification_timeout=verification_timeout, |
| ) |
| presentation.properties.update(cipd_pin._asdict()) |
| if search_tag: |
| # Return search_tag to output properties so it can be |
| # used by builders like goma_toolchain. |
| presentation.properties.update(search_tag) |
| |
| return cipd_pin.instance_id |
| |
| # TODO(https://crbug.com/903435): This will become unnecessary if and when |
| # CIPD supports resolving symlinks that point outside the package root. |
| def hardlink_tree(self, root: config_types.Path) -> HardlinkTree: |
| """Creates a HardlinkTree, given a root directory. |
| |
| This is useful for constructing a self-contained tree of files that can |
| be uploaded to CIPD, copied from a tree of files that may contain |
| symlinks outside the tree root, which the cipd CLI doesn't support. |
| |
| Args: |
| root: root of a tree of hardlinks. |
| """ |
| # Pass in the file module to include all necessary deps for SymlinkTree |
| # which HardlinkTree is a child of. |
| return HardlinkTree(root, self.m.file.m, self.resource("hardlink.py")) |
| |
| def upload_from_manifest( |
| self, |
| cipd_package: str, |
| cipd_manifest: CIPDUploadManifest, |
| build_dir: config_types.Path, |
| repository: Optional[str], |
| git_revision: str, |
| upload_to_cipd: bool, |
| cas_digests: Dict[str, str], |
| ) -> None: |
| """Upload files from cipd_manifest to CIPD as cipd_package. |
| |
| Args: |
| cipd_package: Destination of the CIPD package. |
| cipd_manifest: Files to include in the package. |
| build_dir: Prefix for the paths in cipd_manifest. |
| repository: Remote URL for checkout, used as CIPD metadata. |
| git_revision: Commit hash of checkout, used as CIPD search tag. |
| upload_to_cipd: Whether to upload to CIPD (alternatively uploads to CAS). |
| cas_digests: Output location to put hash from CAS. |
| """ |
| staging_dir = self.m.path.mkdtemp("cipd") |
| tree = self.hardlink_tree(staging_dir) |
| |
| for f in cipd_manifest.files: |
| # We should generally not be uploading artifacts from outside the build |
| # directory, in order to ensure everything flows through the |
| # checkout->build->upload pipeline. So disallow uploading files from |
| # outside the build directory. |
| # |
| # Links to files outside the build directory are still allowed. |
| if ".." in f.source.split("/"): |
| raise self.m.step.StepFailure( |
| f"CIPD upload file source must within the build directory: {f.source}" |
| ) |
| abs_source = build_dir.join(*f.source.split("/")) |
| |
| # For convenience, projects can specify dest="." to upload an entire |
| # directory as the contents of the package. |
| if f.dest == ".": |
| if len(cipd_manifest.files) > 1: # pragma: no cover |
| raise self.m.step.StepFailure( |
| "Only one CIPD manifest entry is allowed if any entry's destination is '.'" |
| ) |
| # No need for a tree of symlinks anymore, we can just treat the |
| # source directory as the staging directory. |
| staging_dir = abs_source |
| tree = None |
| break |
| |
| abs_dest = tree.root.join(*f.dest.split("/")) |
| tree.register_link(abs_source, linkname=abs_dest) |
| |
| if tree: |
| tree.create_links("create hardlinks") |
| |
| if upload_to_cipd: |
| self.upload_package( |
| cipd_package, |
| staging_dir, |
| search_tag={"git_revision": git_revision}, |
| repository=repository, |
| ) |
| else: |
| cas_digests[cipd_package] = self.m.cas_util.upload(staging_dir) |
