recipe_modules/cas_util/api.py

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import datetime

from recipe_engine import recipe_api

from RECIPE_MODULES.recipe_engine.file.api import SymlinkTree


class ArchiveTree:
    """A representation of a tree to archive in RBE-CAS.

    This tree uses the same interface as api.file.SymlinkTree.
    """

    def __init__(self, root, api):
        """See ArchiveApi.tree for the public constructor."""
        self._tree = api.cas_util.hardlink_tree(root)
        self._api = api

    @property
    def root(self):
        """The root (Path) of the tree."""
        return self._tree.root

    def register_link(self, target, linkname):
        """Registers a pair of paths to link."""
        self._tree.register_link(
            self._api.path.abs_to_path(self._api.path.realpath(target)),
            linkname,
        )

    def create_links(self, name, preserve_symlinks=False):
        """Creates all registered links on disk."""
        # Use an env var instead of a flag to pass `preserve_symlinks` so we can
        # call SymlinkTree.create_links as usual.
        # TODO(crbug.com/1216363): this should no longer be necessary once the
        # `cas` CLI respects symlinks.
        env = {}
        if preserve_symlinks:
            env["PRESERVE_SYMLINKS"] = "1"
        with self._api.context(env=env):
            self._tree.create_links(name)


class HardlinkTree(SymlinkTree):
    """A representation of a tree of hardlinks.

    This tree differs from the SymlinkTree in the resource passed to it.
    See ArchiveApi.hardlink_tree for the public constructor.
    """

    pass


class CasUtilApi(recipe_api.RecipeApi):
    """API for archiving and accessing archived files in RBE-CAS."""

    def upload(
        self,
        staging_dir,
        upload_paths=(),
        step_name=None,
        output_property=None,
    ):
        """Returns the digest of the archived tree created from the provided
        staging_dir.

        Args:
            staging_dir (Path): The root of the directory to upload.
            upload_paths (list(Path)): A list of files under the staging_dir
                to upload if not uploading the entire directory.
            output_property (str): The name of the output property to set with
                the digest.
        """

        def do_upload(step_name):
            # TODO(crbug.com/1279920): Remove retries after "digest mismatch"
            # upload failures have been root-caused. The CAS CLI should be able
            # to recover from transient errors internally.
            return self.m.utils.retry(
                lambda: self.m.cas.archive(
                    step_name,
                    staging_dir,
                    *upload_paths,
                    # TODO(olivernewman): Stop setting log_level after slow CAS
                    # uploads have been diagnosed.
                    log_level="debug",
                    # TODO(fxbug.dev/122153): Stop setting a timeout once CAS
                    # slowness is resolved and/or the CAS CLI sets shorter
                    # internal timeouts.
                    timeout=datetime.timedelta(minutes=15),
                ),
                max_attempts=3,
            )

        # TODO(fxbug.dev/62939): The two codepaths below mirror what was in
        # upload_isolated before the migration happened so that step names stay
        # the same. Combine the two paths once we confirm there are no
        # dependencies on the step names.
        if output_property:
            with self.m.step.nest(step_name or "cas") as step:
                cas_digest = do_upload("archive")
                step.presentation.properties[output_property] = cas_digest
        else:
            cas_digest = do_upload(step_name or "archive")
        return cas_digest

    def download(self, digest, output_dir, step_name=None):
        """Downloads the archive with the CAS digest into output_dir."""
        self.m.cas.download(
            step_name or "download", digest=digest, output_dir=output_dir
        )

    def tree(self, root):
        """Creates an ArchiveTree, given a root directory.

        Args:
            root (Path): root of a tree to archive.
        """
        return ArchiveTree(root, self.m)

    def hardlink_tree(self, root):
        """Creates a HardlinkTree, given a root directory.

        Args:
            root (Path): root of a tree of hardlinks.
        """
        # Pass in the file module to include all necessary deps for SymlinkTree
        # which HardlinkTree is a child of.
        return HardlinkTree(root, self.m.file.m, self.resource("hardlink.py"))