# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import functools

from recipe_engine import recipe_api
from urlparse import urlparse

from . import patch
from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2

# Set as an output property and consumed by other recipes code and the results
# uploader code in google3.
# It's a monotonic integer that corresponds to integration revisions so we
# can stuff our results into systems that expect Piper changelist numbers.
REVISION_COUNT_PROPERTY = "integration-revision-count"

# Set as an output property to be used in the coverage recipe to remap source
# file paths to a new checkout.
ROOT_DIR_PROPERTY = "checkout_root"

# By default, skip patching GerritChanges which map to these projects. They are
# not valid projects to patch into standard Fuchsia checkouts.
SKIP_PATCH_PROJECTS = ("infra/recipes",)


class _CheckoutResults(object):
    """Represents a Fuchsia source checkout."""

    def __init__(
        self,
        api,
        root_dir,
        snapshot_file,
        release_branch,
        release_version,
        source_info,
    ):
        self._api = api
        self._root_dir = root_dir
        self._snapshot_file = snapshot_file
        self._release_branch = release_branch
        self._release_version = release_version
        self.source_info = source_info
        self._changed_files_cache = None

    @property
    def root_dir(self):
        """The path to the root directory of the jiri checkout."""
        return self._root_dir

    @property
    def snapshot_file(self):
        """The path to the jiri snapshot file."""
        return self._snapshot_file

    @property
    def release_branch(self):
        """Release branch corresponding to checkout if applicable, otherwise None."""
        return self._release_branch

    @property
    def release_version(self):
        """Release version of checkout if applicable, otherwise None."""
        return self._release_version

    def project(self, project_name, **kwargs):
        return self._api.checkout.project(
            project_name, checkout_root=self.root_dir, **kwargs
        )

    @property
    def integration_revision(self):
        # If triggered by integration CQ, then build_input_resolver will ensure
        # the input gitiles_commit is what we want.
        if (
            self._api.buildbucket.build.input.gerrit_changes
            and self._api.buildbucket.build.input.gerrit_changes[0].project
            == "integration"
        ):
            return self._api.buildbucket.build.input.gitiles_commit.id
        # Otherwise just use the revision that we actually checked out.
        return [repo for repo in self.source_info if repo["name"] == "integration"][0][
            "revision"
        ]

    def _upload_source_manifest(self, gcs_bucket, namespace=None):
        """Upload the jiri source manifest to GCS."""
        assert gcs_bucket
        with self._api.context(cwd=self._root_dir):
            source_manifest = self._api.jiri.source_manifest()
        with self._api.step.nest("upload source manifest"):
            self._api.upload.file_to_gcs(
                source=self._api.json.input(source_manifest),
                bucket=gcs_bucket,
                subpath="source_manifest.json",
                namespace=namespace,
            )

    def upload_results(self, gcs_bucket, namespace=None):
        """Upload snapshot to a given GCS bucket."""
        assert gcs_bucket
        with self._api.step.nest("upload checkout results") as presentation:
            self._api.upload.file_to_gcs(
                source=self.snapshot_file,
                bucket=gcs_bucket,
                subpath=self._api.path.basename(self.snapshot_file),
                namespace=namespace,
            )
            presentation.properties[REVISION_COUNT_PROPERTY] = int(
                self._api.git(
                    "set %s property" % REVISION_COUNT_PROPERTY,
                    "-C",
                    self._root_dir.join("integration"),
                    "rev-list",
                    "--count",
                    "HEAD",
                    step_test_data=lambda: self._api.raw_io.test_api.stream_output("1"),
                    stdout=self._api.raw_io.output(),
                ).stdout.strip()
            )
            if not self._api.platform.is_mac:
                self._upload_source_manifest(gcs_bucket, namespace=namespace)

    def changed_files(
        self, buildbucket_input, test_data=("foo.cc", "bar.cc"), **kwargs
    ):
        """Returns a list of absolute paths that were changed.

        Checks the git repo specified in buildbucket_input.gerrit_changes[0].

        Args:
            buildbucket_input (build_pb2.Input): The buildbucket input from
                which to retrieve CL info.
            test_data (seq of str): Mock list of changed files.
            **kwargs (dict): Passed through to `api.git.get_changed_files()`.

        Returns:
            Empty list if buildbucket_input.gerrit_changes is empty.
            list of paths otherwise.
        """
        if not buildbucket_input.gerrit_changes:
            return []

        change = buildbucket_input.gerrit_changes[0]
        cache_key = (
            change.host,
            change.project,
            change.change,
            change.patchset,
            kwargs,
        )
        if not self._changed_files_cache or cache_key != self._changed_files_cache[0]:
            with self._api.step.nest("get changed files"):
                project = change.project
                with self._api.context(cwd=self._root_dir):
                    project_test_data = [
                        {
                            "name": project,
                            "path": self._api.path.abspath(self._root_dir)
                            if project == "project"
                            else self._api.path.abspath(self._root_dir.join(project)),
                        }
                    ]
                    try:
                        repo_path = self.project(project, test_data=project_test_data)[
                            "path"
                        ]
                    except self._api.jiri.NoSuchProjectError:
                        return []
                with self._api.context(cwd=self._api.path.abs_to_path(repo_path)):
                    changed_files = self._api.git.get_changed_files(
                        test_data=test_data, **kwargs
                    )
                changed_files = [
                    self._api.path.join(repo_path, changed) for changed in changed_files
                ]
                # We only expect this function to be called with one key per build, so
                # keeping a cache of one element should be sufficient, while still
                # being correct in case it is called with different keys.
                self._changed_files_cache = (cache_key, changed_files)
        return self._changed_files_cache[1]


def _nest(func):
    """Nest function call within "checkout" step.

    Check whether already inside a "checkout" step since some public
    methods in CheckoutApi call other public methods.
    """

    @functools.wraps(func)
    def wrapper(self, *args, **kwargs):
        if not self._nested:
            with self.m.step.nest("checkout"):
                self._nested = True
                ret = func(self, *args, **kwargs)
                self._nested = False
                return ret
        else:
            return func(self, *args, **kwargs)

    return wrapper


class CheckoutApi(recipe_api.RecipeApi):
    """An abstraction over how Jiri checkouts are created during Fuchsia CI/CQ
    builds."""

    REVISION_COUNT_PROPERTY = REVISION_COUNT_PROPERTY
    ROOT_DIR_PROPERTY = ROOT_DIR_PROPERTY

    # An invalid patch file represents a user error not an infra
    # failure. A user should correct their change.
    class PatchFileValidationError(recipe_api.StepFailure):
        pass

    def __init__(self, buildset, *args, **kwargs):
        super(CheckoutApi, self).__init__(*args, **kwargs)
        self._nested = False
        self._gitiles_commit = None
        # If buildset != None, it will be of the format
        # commit/gitiles/host/project/+/revision.
        # If provided, it will override the gitiles_commit from the build input used
        # in the checkout. This would happen in the case that the build input commit
        # is tracking a different repo than the one we want to checkout (as in the
        # fuchsia builds launched by the toolchain recipes).
        if buildset:
            buildset = buildset.replace("commit/gitiles/", "")
            buildset_parts = buildset.split("/")
            host = buildset_parts[0]
            project = "/".join(buildset_parts[1:-2])
            revision = buildset_parts[-1]
            self._gitiles_commit = common_pb2.GitilesCommit(
                host=host, project=project, id=revision,
            )

    def CheckoutResults(
        self, root_dir, snapshot_file, release_branch, release_version, source_info
    ):
        """Return a CheckoutResults object.

        Outside this module, should only be used in testing example recipes.
        """
        return _CheckoutResults(
            self.m,
            root_dir,
            snapshot_file,
            release_branch,
            release_version,
            source_info,
        )

    @_nest
    def with_options(self, *args, **kwargs):
        with self.m.context(infra_steps=True):
            return self._with_options(*args, **kwargs)

    def _with_options(
        self,
        path,
        manifest,
        remote,
        project=None,
        patch_project=None,
        attributes=(),
        build_input=None,
        run_hooks=True,
        fetch_packages=True,
        skip_patch_projects=SKIP_PATCH_PROJECTS,
        timeout_secs=None,
    ):
        """Initializes and populates a jiri checkout from a remote manifest.

        Emits a source info for the build.

        Args:
            path (Path): The Fuchsia checkout root.
            manifest (str): Relative path to the manifest in the remote repository.
            remote (str): URL to the remote repository.
            project (str): The name that jiri should assign to the project.
            patch_project (str): The name of the jiri project to patch if different
                from the gerrit_change project.
            attributes (seq(str)): A list of jiri manifest attributes; projects or
                packages with matching attributes - otherwise regarded as optional -
                will be downloaded.
            build_input (buildbucket.build_pb2.Build.Input): The input to a buildbucket
                build.
            run_hooks (bool): Whether or not to run the hooks.
            fetch_packages (bool): Whether or not to fetch CIPD packages.
            skip_patch_projects (seq(str)): Do not attempt to patch these
                projects.
            timeout_secs (int): A timeout for jiri update in seconds.

        Returns:
            A Python dictionary representing the source info.
        """
        build_input = build_input or self.m.buildbucket.build.input

        if build_input.gerrit_changes:
            gerrit_change = build_input.gerrit_changes[0]
            rest = build_input.gerrit_changes[1:]
            assert rest == [], "build information contains more than one patchset"
            self.m.jiri.init(
                directory=path,
                attributes=attributes,
                skip_partial=[
                    "https://%s/%s"
                    % (
                        gerrit_change.host.replace("-review", ""),
                        patch_project or gerrit_change.project,
                    )
                ],
            )
            self.from_patchset(
                path=path,
                manifest=manifest,
                remote=remote,
                project=project,
                patch_project=patch_project,
                run_hooks=run_hooks,
                fetch_packages=fetch_packages,
                timeout_secs=timeout_secs,
                gerrit_change=gerrit_change,
                skip_patch_projects=skip_patch_projects,
            )
        else:
            self.m.jiri.init(directory=path, attributes=attributes)
            commit = None
            if build_input and build_input.gitiles_commit.id:
                commit = build_input.gitiles_commit

            if self._gitiles_commit:
                # Use gitiles_commit from properties instead of build input.
                commit = self._gitiles_commit

            self.from_commit(
                path=path,
                manifest=manifest,
                remote=remote,
                commit=commit,
                project=project,
                run_hooks=run_hooks,
                fetch_packages=fetch_packages,
                timeout_secs=timeout_secs,
            )

        with self.m.context(cwd=path):
            return self.m.jiri.project(name="source-info").json.output

    @_nest
    def from_spec(self, checkout_spec, checkout_root=None):
        """Initialize a Fuchsia checkout according to a checkout spec.

        If a gitiles_commit was provided through the buildset property, this will
        set the buildbucket.build.input's gitiles_commit to the gitiles_commit from
        the property.

        Args:
          checkout_spec: infra.fuchsia.Fuchsia.Checkout protocol buffer message.
          checkout_root: the path to checkout into.
        """
        build_input = self.m.buildbucket.build.input
        if self._gitiles_commit:
            # Override build input gitiles_commit with gitiles_commit from
            # properties.
            build_input.gitiles_commit.CopyFrom(self._gitiles_commit)
        if not checkout_root:
            checkout_root = self.m.path["start_dir"].join("fuchsia")
            if self.m.experimental.incremental_build:
                checkout_root = self.m.path["cache"].join("incremental")
        assert checkout_spec.manifest
        assert checkout_spec.remote
        checkout = self.fuchsia_with_options(
            path=checkout_root,
            build_input=build_input,
            manifest=checkout_spec.manifest,
            remote=checkout_spec.remote,
            attributes=checkout_spec.attributes,
            is_release_version=checkout_spec.is_release_version,
        )

        return checkout

    @_nest
    def from_patchset(
        self,
        path,
        manifest,
        remote,
        project,
        patch_project,
        run_hooks,
        fetch_packages,
        timeout_secs,
        gerrit_change,
        skip_patch_projects=SKIP_PATCH_PROJECTS,
    ):
        """Initializes and populates a Jiri checkout from a remote manifest and Gerrit
        change.

        Args:
            path (Path): The Fuchsia checkout root.
            manifest (str): Relative path to the manifest in the remote repository.
            remote (str): URL to the remote repository.
            project (str): The name that jiri should assign to the project.
            patch_project (str): The name of the jiri project to patch if different
                from the gerrit_change project.
            run_hooks (bool): Whether or not to run the hooks.
            fetch_packages (bool): Whether or not to fetch CIPD packages.
            timeout_secs (int): A timeout for jiri update in seconds.
            gerrit_change (GerritChange): Change information derived from
                build.input.gerrit_change.
            skip_patch_projects (seq(str)): Do not attempt to patch these
                projects.
        """
        is_integration_patch = project == gerrit_change.project

        # Fetch the project and update.
        change_details = self._get_change_details(gerrit_change)
        with self.m.context(cwd=path):
            self.m.jiri.import_manifest(
                manifest,
                remote,
                overwrite=self.m.experimental.incremental_build,
                name=project,
                # Always re-resolve HEAD rather than using the base commit
                # resolved by build_input_resolver, even for integration
                # patches, because we want to wait as late as possible to
                # choose a base commit for the checkout. This will make it more
                # likely that we'll catch merge conflicts and other types of
                # collisions between changes.
                #
                # The most notable risk of doing this is that the spec might
                # come from a different version of the integration repo than
                # the version included in this checkout, which could cause
                # issues if the spec is not forwards-compatible with newer
                # versions of the integration repo (e.g. it references a jiri
                # project that is deleted from the manifests). However, this is
                # extremely rare in practice because since we rarely make
                # changes that simultaneously affect infra and non-infra parts
                # of integration like jiri manifests.
                revision=self.m.git.get_remote_branch_head(
                    url=self.m.sso.sso_to_https(remote),
                    branch=change_details["branch"],
                    step_name="resolve HEAD",
                ),
            )

            self.m.jiri.update(
                run_hooks=False,
                fetch_packages=False,
                timeout=timeout_secs,
                gc=self.m.experimental.incremental_build,
            )

            # Patch the current Gerrit change.
            current_revision = self._get_current_revision(gerrit_change, change_details)
            patch_ref = current_revision["ref"]

            patch_project = patch_project or gerrit_change.project

            # Failures in pulling down patches and rebasing are likely not
            # infra-related. If we got here, we're already able to talk to Gerrit
            # successfully, so any errors are likely merge conflicts.
            with self.m.context(infra_steps=False):
                if patch_project not in skip_patch_projects:
                    self._apply_patch(
                        path,
                        patch_ref,
                        patch_project,
                        gerrit_change,
                        change_details,
                        is_integration_patch,
                        timeout_secs,
                    )

            if run_hooks:
                self.m.jiri.run_hooks(
                    local_manifest=True, fetch_packages=fetch_packages
                )
                if fetch_packages:
                    self.m.jiri.fetch_packages(local_manifest=True)

    @_nest
    def from_commit(
        self,
        path,
        manifest,
        remote,
        commit,
        project,
        run_hooks,
        fetch_packages,
        timeout_secs,
    ):
        """Populates a Jiri checkout from a remote manifest and Gitiles commit.

        Args:
            path (Path): The Fuchsia checkout root.
            manifest (str): Relative path to the manifest in the remote repository.
            remote (str): URL to the remote repository.
            commit (GitilesCommit): Commit information derived from
                build.input.gitiles_commit.
            project (str): The name that jiri should assign to the project.
            run_hooks (bool): Whether or not to run the hooks.
            fetch_packages (bool): Whether or not to fetch CIPD packages.
            timeout_secs (int): A timeout for jiri update in seconds.
        """
        revision = "HEAD"
        override = False

        if commit:
            revision = commit.id
            # If the commit project != manifest project, then we get the manifest at
            # HEAD and override only the commit's project.
            override = commit.project != project

        with self.m.context(cwd=path):
            if override:
                self.m.jiri.import_manifest(
                    manifest,
                    remote,
                    name=project,
                    revision="HEAD",
                    overwrite=self.m.experimental.incremental_build,
                )

                # In order to identify a project to override, jiri keys on
                # both the project name and the remote source repository (not to be
                # confused with `remote`, the manifest repository).
                # Doing this correctly would require finding the commit's remote in the
                # transitive imports of the jiri manifest. But those transitive imports
                # aren't available until we run "jiri update", and doing that twice is
                # slow, so we rely on:
                # 1. The convention that the name of the jiri project
                #    is the same as commit.project.
                # 2. The hope that the URL scheme of the commit remote is the same as that
                #    of the manifest remote.
                manifest_remote_url = urlparse(remote)
                host = commit.host
                # When using sso we only specify the lowest subdomain, by convention.
                if manifest_remote_url.scheme == "sso":
                    host = host.split(".")[0]
                commit_remote = "%s://%s/%s" % (
                    manifest_remote_url.scheme,
                    host,
                    commit.project,
                )
                self.m.jiri.override(
                    project=commit.project,  # See 1. above
                    remote=commit_remote,
                    new_revision=revision,
                )
            else:
                self.m.jiri.import_manifest(
                    manifest,
                    remote,
                    name=project,
                    revision=revision,
                    overwrite=self.m.experimental.incremental_build,
                )

            self.m.jiri.update(
                run_hooks=False,
                fetch_packages=fetch_packages,
                timeout=timeout_secs,
                gc=self.m.experimental.incremental_build,
            )
            if run_hooks:
                self.m.jiri.run_hooks(fetch_packages=fetch_packages)

    @_nest
    def fuchsia_with_options(
        self,
        path,
        manifest,
        remote,
        project="integration",
        is_release_version=False,
        timeout_secs=45 * 60,
        **kwargs
    ):
        """Uses Jiri to check out a Fuchsia project.

        The root of the checkout is returned via _CheckoutResults.root_dir.

        Args:
            path (Path): The Fuchsia checkout root.
            manifest (str): A path to the manifest in the remote (e.g. manifest/minimal)
            remote (str): A URL to the remote repository which Jiri will be pointed at
            is_release_version (bool): Whether the checkout is a release version.
            timeout_secs (int): How long to wait for the checkout to complete
                before failing
            **kwargs (dict): Passed through to with_options().

        Returns:
            A _CheckoutResults containing details of the checkout.
        """
        source_info = self.with_options(
            path, manifest, remote, project=project, timeout_secs=timeout_secs, **kwargs
        )

        with self.m.context(infra_steps=True, cwd=path):
            snapshot_file = self.m.path["cleanup"].join("jiri_snapshot.xml")
            self.m.jiri.snapshot(snapshot_file)
            return self.finalize_checkout(
                path=path,
                snapshot_file=snapshot_file,
                is_release_version=is_release_version,
                source_info=source_info,
            )

    def _apply_patch(
        self,
        path,
        patch_ref,
        patch_project,
        gerrit_change,
        change_details,
        is_integration_patch,
        timeout_secs,
    ):
        rebase_branch = change_details["branch"]
        self.m.jiri.patch(
            patch_ref,
            host="https://%s" % gerrit_change.host,
            project=patch_project,
            rebase=True,
            rebase_branch=rebase_branch,
        )

        # Handle patches.json, if present.
        self._apply_patchfile(path, gerrit_change, rebase_branch)

        if is_integration_patch:
            self.m.jiri.update(
                gc=True,
                rebase_tracked=True,
                local_manifest=True,
                run_hooks=False,
                fetch_packages=False,
                timeout=timeout_secs,
            )

        # It's difficult to figure out what commit the tryjob rebased a CL on
        # top of. So we simply log the last few commits here. (It's not
        # sufficient to log just the parent commit, because checking out a CL
        # at the top of a stack of open CLs will also check out and rebase all
        # the parent CLs on top of master).
        project_dir = self.project(patch_project)["path"]
        self.m.git.log(directory=project_dir, depth=10)

    def _apply_patchfile(self, path, gerrit_change, rebase_branch):
        """Parses and applies the PatchFile for the given gerrit change."""
        # Verify patches.json exists.
        # TODO: This is a fragile assumption that relies on integration.git
        # being checked out at //integration. Find a better way to derive path
        # to patches.json.
        patchfile_path = path.join(gerrit_change.project, "patches.json")
        if not self.m.path.exists(patchfile_path):
            return

        patch_file = self._parse_patchfile(patchfile_path)

        # Ensure patchfile is valid.
        validation_err = patch_file.validate(gerrit_change)
        if validation_err is not None:
            raise self.PatchFileValidationError(str(validation_err))

        for patch_input in patch_file.inputs:
            # If the patch pulls in a project that's not in the workspace already, the patch
            # would not affect this build / test run.  Skip this patch.
            try:
                self.project(patch_input.project)
            except self.m.jiri.NoSuchProjectError:
                warning = (
                    "warning: skipping patch for %s which is not in the checkout"
                    % patch_input.project
                )
                self.m.step(warning, [])
                continue

            # Strip protocol if present.
            host = patch_input.host
            host_url = urlparse(host)
            if host_url.scheme:
                host = host_url.hostname

            # Patch in the change
            self.m.jiri.patch(
                ref=patch_input.ref,
                host="https://%s" % host,
                project=patch_input.project,
                rebase=True,
                rebase_branch=rebase_branch,
            )

    def _get_change_details(self, gerrit_change):
        """Fetches the details of a Gerrit change."""
        return self.m.gerrit.change_details(
            name="get change details",
            change_id="%s~%s" % (gerrit_change.project, gerrit_change.change,),
            host=gerrit_change.host,
            query_params=["ALL_REVISIONS"],
            test_data=self.m.json.test_api.output(
                {
                    "branch": "master",
                    "revisions": {
                        "d4e5f6": {"_number": 3, "ref": "refs/changes/00/100/3"},
                        "a1b2c3": {"_number": 7, "ref": "refs/changes/00/100/7"},
                        "g7h8i9": {"_number": 9, "ref": "refs/changes/00/100/9"},
                    },
                }
            ),
        ).json.output

    def _get_current_revision(self, gerrit_change, change_details):
        current_patchsets = [
            rev
            for rev in change_details["revisions"].itervalues()
            if rev["_number"] == gerrit_change.patchset
        ]
        assert len(current_patchsets) == 1
        return current_patchsets[0]

    def _parse_patchfile(self, patchfile_path):
        """Parses a PatchFile from the given path."""
        js = self.m.file.read_json("read patches-json", patchfile_path)
        return patch.PatchFile.from_json(js)

    def _get_release_version(self, path):
        """Get release version corresponding to HEAD."""
        with self.m.step.nest("resolve release version") as presentation:
            release_version = self.m.release.ref_to_release_version(
                ref="HEAD", repo_path=path.join("integration"),
            )
            release_version_str = str(release_version).replace("releases/", "")
            presentation.properties["release_version"] = release_version_str
        return release_version_str

    def _get_release_branch(self, path):
        with self.m.step.nest("resolve release branch") as presentation:
            ref = self.m.buildbucket.build.input.gitiles_commit.ref.replace(
                "refs/heads/", ""
            )
            if self.m.release.validate_branch(ref):
                presentation.properties["release_branch"] = ref
                return ref

    def finalize_checkout(self, path, snapshot_file, is_release_version, source_info):
        """Finalizes a checkout; constructs a _CheckoutResults object.

        Args:
          path (Path): The Fuchsia checkout root.
          snapshot_file (Path): Path to the snapshot file.
          is_release_version (bool): Whether this checkout is a release version.
          source_info (dict): A Python dictionary representing the source info.

        Returns:
          A _CheckoutResults containing details of the checkout.
        """
        # Always log snapshot contents (even if uploading to GCS) to help debug
        # things like tryjob failures during roller-commits.
        self.m.file.read_text("read snapshot", snapshot_file)

        # If using a release version, resolve release version and release
        # branches.
        release_version = release_branch = None
        if is_release_version:
            release_version = self._get_release_version(path)
            release_branch = self._get_release_branch(path)

        return self.CheckoutResults(
            root_dir=path,
            snapshot_file=snapshot_file,
            release_branch=release_branch,
            release_version=release_version,
            source_info=source_info,
        )

    def project(self, project_name, checkout_root=None, **kwargs):
        """Returns metadata for a project in the checkout.

        Raises NoSuchProjectError if the project is not among the repos in
        the checkout.
        """
        if not checkout_root:
            checkout_root = self.m.context.cwd
        with self.m.context(cwd=checkout_root):
            output = self.m.jiri.project(projects=[project_name], **kwargs).json.output
        if not output:
            raise self.m.jiri.NoSuchProjectError(
                "project %r is not present in the checkout" % project_name
            )
        return output[0]