# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for rolling packages and projects to versions based on data in CIPD.

Packages that depend on flutter need to get rolled based on the version
dependencies of another fuchsia prebuilt. In this recipe we are
calling the fuchsia prebuilt dictating the dependencies "orchestrator".

The orchestrator cipd package contains a json file with the dependencies
it was built with. The following is an example of the json file:

{
 "flutter_version": "flutter_version_xyz",
 "engine_version": "engine_version_xyz",
 "skia_version": "skia_version_xyz",
 "dart_version": "dart_version_xyz"
}

This recipe checks if there is a new version of the orchestrator and
if it requires different versions of the dependencies from the ones in
the source tree. If both conditions are true then it proceeds to roll
new versions of projects and packages as described in the
rolling_packages_projects parameter and using the versions from the
orchestrator dependencies json file.

Warning: If this recipe runs in dry mode it will still roll changes into
fuchsia dart 3p repository but the fuchsia tree won't pin to the latest
version. This is safe as the fuchsia dart 3p hash will be bumped only on a
successful roll of all the other dependencies.

TODO(msteen@google.com): This contains a lot of legacy logic for rolling flutter
itself. This is confusing and should probably get cleaned up.
"""

from recipe_engine.config import List
from recipe_engine.recipe_api import Property

DEPS = [
    "fuchsia/auto_roller",
    "fuchsia/buildbucket_util",
    "fuchsia/cipd_dependencies",
    "fuchsia/debug_symbols",
    "fuchsia/git",
    "fuchsia/jiri",
    "fuchsia/python3",
    "recipe_engine/buildbucket",
    "recipe_engine/cipd",
    "recipe_engine/context",
    "recipe_engine/file",
    "recipe_engine/json",
    "recipe_engine/path",
    "recipe_engine/properties",
    "recipe_engine/step",
]

PROPERTIES = {
    "project": Property(kind=str, help="Jiri remote manifest project", default=None),
    "manifests": Property(
        kind=List(dict),
        help=(
            "A list of dictionaries with project, manifest and remote keys "
            "used to signal jiri wich manifests to synchronize. "
            'E.g. [{"project": "integration", "manifest": "other/dependency", '
            '      "remote": "sso://fuchsia/integration", '
            '      "lock_file": "integration/jiri.lock"}]'
        ),
    ),
    "locks": Property(
        kind=List(dict),
        help=(
            "A list of dictionaries with manifest and files keys used to "
            "signal jiri which lock files to update. "
            ' E.g. [{"manifest": "integration/fuchsia/flower", '
            '        "file": "integration/fuchsia/jiri.lock"}]'
        ),
    ),
    "orchestrator_package": Property(
        kind=str,
        help=(
            "A string with the CIPD package used to validate "
            "flutter and dependencies versions. This is a special "
            "cipd package that contains a json file describing "
            "the dependencies it was built with."
        ),
    ),
    "orchestrator_ref": Property(
        kind=str,
        default="latest",
        help="A cipd ref used to get a given version of the orchestrator package",
    ),
    "orchestrator_import_in": Property(
        kind=str,
        default="",
        help=(
            "A string with the location where the orchestrator package "
            "should be imported into the source tree."
        ),
    ),
    "owners": Property(
        kind=List(str),
        default=(),
        help=(
            "The owners responsible for watching this roller "
            '(example: "username@google.com").'
        ),
    ),
    "rolling_packages_projects": Property(
        kind=List(dict),
        default=(),
        help=(
            "A list of dictionaries with rolling package/project, type,"
            " and location to import_in. This is used to list the "
            "dependencies that should be updated: "
            'E.g. [{"name": "dart", "type": "package", '
            '       "import_in": "fuchsia/prebuilts", '
            '       "version_tag": "dart_version"}]'
        ),
    ),
    "test_multipliers": Property(
        kind=List(dict),
        default=None,
        help=("A list of test multipliers to pass into the roll CL MULTIPLY footer."),
    ),
    "versions_file": Property(
        kind=str,
        help=(
            "A string with the path to the versions file inside"
            "the orchestrator CIPD package."
        ),
    ),
    "validate_against_package": Property(
        kind=dict,
        help=(
            "A dictionary with the package and tag for the version to use"
            " for validation. A single dependency is used to validate if"
            ' a roll is needed. E.g. {"package": "flutter/fuchsia", '
            '                          "version_tag": "flutter_version"} '
            "will validate the source tree version of flutter/fuchsia "
            "with the value of flutter_version from the orchestrator "
            "json file."
        ),
    ),
    "dry_run": Property(
        kind=bool,
        default=False,
        help="Whether to dry-run the auto-roller (CQ+1 and abandon the change)",
    ),
    "debug_symbol_packages": Property(
        kind=List(str),
        default=(),
        help=(
            "A list of strings with the cipd packages containing debug "
            "symbols and their associated packages."
        ),
    ),
    "debug_symbol_gcs_buckets": Property(
        kind=List(str),
        default=(),
        help="GCS buckets to upload debug symbols to",
    ),
    "dart_pkg_manifest": Property(
        kind=str,
        help="Location to dart package manifest",
    ),
}

COMMIT_MESSAGE = """[roll] Roll {roller} packages and projects

{packages}

{multiply}Test: CQ"""


def RunSteps(
    api,
    project,
    manifests,
    locks,
    orchestrator_ref,
    owners,
    versions_file,
    rolling_packages_projects,
    orchestrator_package,
    orchestrator_import_in,
    validate_against_package,
    dry_run,
    debug_symbol_packages,
    debug_symbol_gcs_buckets,
    dart_pkg_manifest,
    test_multipliers,
):
    with api.context(infra_steps=True):
        if owners:
            with api.step.nest("owners") as presentation:
                presentation.step_summary_text = ", ".join(owners)

        # Import manifests
        with api.step.nest("jiri-manifests") as presentation:
            api.jiri.init(use_lock_file=True)
            presentation.logs["manifests"] = api.json.dumps(manifests)
            for manifest_config in manifests:
                api.jiri.import_manifest(
                    manifest=manifest_config["manifest"],
                    remote=manifest_config["remote"],
                    name=manifest_config["project"],
                )
            api.jiri.update(run_hooks=False)
            with api.context(cwd=api.path["start_dir"]):
                api.jiri.run_hooks()

    deps_info = api.jiri.package([orchestrator_package])
    cipd_description = api.cipd.describe(orchestrator_package, orchestrator_ref)
    orchestrator_current_version = deps_info.json.output[0]["version"]
    # Is there a new version to roll?
    orchestrator_expected_version = cipd_description.tags[0].tag
    if orchestrator_expected_version == orchestrator_current_version:
        msg = "%s in tree version is [%s] and latest from cipd is [%s]" % (
            orchestrator_package,
            orchestrator_current_version,
            orchestrator_expected_version,
        )
        api.step.empty("manifest up-to-date; nothing to roll", step_text=msg)
        return api.auto_roller.nothing_to_roll()

    # Read main package versions file.
    orchestrator_version_dict = api.cipd_dependencies.get_dependencies(
        "read versions",
        orchestrator_package,
        cipd_description.pin.instance_id,
        versions_file,
    )

    # Use version of the validation package to decide whether a roll is needed or
    # not.
    validation_info = api.jiri.package([validate_against_package["package"]])
    local_version = validation_info.json.output[0]["version"]
    expected_version = (
        "git_revision:%s"
        % orchestrator_version_dict[validate_against_package["version_tag"]]
    )
    if local_version == expected_version:
        msg = "Expected version [%s] == local fuchsia tree version [%s]" % (
            expected_version,
            local_version,
        )
        api.step.empty("validation summary", step_text=msg)
        return api.auto_roller.nothing_to_roll()
    # Start rolling orchestrator
    rolling_packages_projects.append(
        {
            "type": "package",
            "name": orchestrator_package,
            "version_tag": orchestrator_package,
            "import_in": orchestrator_import_in,
        }
    )
    orchestrator_version_dict[orchestrator_package] = orchestrator_expected_version
    # Roll all the dependencies
    package_msgs = []
    flutter_fuchsia_version = None
    package_msg_template = "%s %s rolled from %s to %s"
    with api.step.nest("edit jiri manifests"):
        for package_project in rolling_packages_projects:
            package_msg = ""
            if package_project["type"] == "package":
                if (
                    package_project["name"] == orchestrator_package
                    or package_project["version_tag"] == "orchestrator_version"
                ):
                    expected_version = orchestrator_version_dict[orchestrator_package]
                else:
                    expected_version = (
                        "git_revision:%s"
                        % orchestrator_version_dict[package_project["version_tag"]]
                    )
                # Save flutter/fuchsia version, as it is the coordinating version
                # for debug symbol uploads.
                if package_project["name"] == "flutter/fuchsia":
                    flutter_fuchsia_version = expected_version
                changes = api.jiri.edit_manifest(
                    package_project["import_in"],
                    packages=[(package_project["name"], expected_version)],
                    name=package_project["name"],
                )
                # Sometimes only a subset of packages are updated. In those cases
                # the changes list is empty.
                if changes["packages"]:
                    old_version = changes["packages"][0]["old_version"]
                    package_msg = package_msg_template % (
                        "package",
                        package_project["name"],
                        old_version,
                        expected_version,
                    )
            else:
                expected_version = orchestrator_version_dict[
                    package_project["version_tag"]
                ]
                changes = api.jiri.edit_manifest(
                    package_project["import_in"],
                    name=package_project["name"],
                    projects=[(package_project["name"], expected_version)],
                )
                # Sometimes only a subset of projects are updated. In those cases
                # the changes list is empty.
                if changes["projects"]:
                    old_version = changes["projects"][0]["old_revision"]
                    package_msg = package_msg_template % (
                        "project",
                        package_project["name"],
                        old_version,
                        expected_version,
                    )
            if package_msg:
                package_msgs.append(package_msg)

    # Update fuchsia's third-party dart packages (including flutter dependencies),
    # only if we are rolling a new version of dart-sdk.
    rolling_package_names = [p["name"] for p in rolling_packages_projects]
    if "fuchsia/dart-sdk/${platform}" in rolling_package_names:
        with api.step.nest("third-party dart packages") as presentation:
            checkout_root = api.path["start_dir"]
            rolled_hash = update_3p_packages(
                api=api,
                presentation=presentation,
                checkout_root=checkout_root,
                flutter_revision=orchestrator_version_dict["flutter_version"],
            )
            changes = api.jiri.edit_manifest(
                dart_pkg_manifest,
                name="third_party/dart-pkg",
                projects=[("third_party/dart-pkg", rolled_hash)],
            )

    # Update lock files
    with api.step.nest("update jiri lock files"):
        for lock in locks:
            api.jiri.resolve(
                local_manifest=True, output=lock["file"], manifests=[lock["manifest"]]
            )

    roller_name = api.buildbucket.builder_name
    suffix = "-roller"
    if roller_name.endswith(suffix):
        roller_name = api.buildbucket.builder_name[: -len(suffix)]

    multiply = ""
    if test_multipliers:
        multiply = "\nMULTIPLY: `%s`\n" % api.json.dumps(test_multipliers, indent=2)

    message = COMMIT_MESSAGE.format(
        roller=roller_name,
        packages="\n\n".join(package_msgs),
        builder=api.buildbucket.builder_name,
        build_id=api.buildbucket_util.id,
        multiply=multiply,
    )

    # All the changes are happening inside one of the git cloned projects.
    # The "project" property points to the location of the project containing
    # the changes and we need to move to that directory for the roller to pick
    # up the changes.
    project_dir = api.path["start_dir"].join(*project.split("/"))
    with api.context(cwd=project_dir):
        change = api.auto_roller.attempt_roll(
            api.auto_roller.Options(
                remote=manifests[0]["remote"],
                dry_run=dry_run,
                roller_owners=owners,
            ),
            repo_dir=project_dir,
            commit_message=message,
        )
        rolled = change and change.success

    # Skip building and pushing Flutter debug symbols if a new version is not
    # provided or running in dry run mode. For dry run mode, dependencies won't
    # be available in the tree.
    if (
        rolled
        and debug_symbol_gcs_buckets
        and flutter_fuchsia_version is not None
        and debug_symbol_packages
        and not dry_run
    ):
        api.debug_symbols.fetch_and_upload(
            packages=debug_symbol_packages,
            version=flutter_fuchsia_version,
            buckets=debug_symbol_gcs_buckets,
        )

    return api.auto_roller.raw_result(change)


def update_3p_packages(
    api, checkout_root, presentation, flutter_revision=None, dry_run=False
):
    """Updates fuchsia's third-party dart packages.

    Args:
        presentation (StepPresentation): A presentation to attach logs, etc. to.
        checkout_root (Path): Root path to checkouts.
        flutter_revision (Optional): A git hash within the flutter repo.
        dry_run (bool): Whether the roll will be allowed to complete or not.

    Returns:
        A string with third_party/dart-pkg/pub latest commit hash.
    """
    commit_msg = "[roll] Update third-party dart packages\n"
    packages_root = checkout_root.join("third_party", "dart-pkg", "pub")
    with api.context(cwd=packages_root):
        # Make sure third_party/dart-pkg is at origin/main before running the
        # update script to catch any manual commits that extend past the revision at
        # integration's HEAD.
        api.git("git fetch", "fetch", "origin")
        api.git("git checkout", "checkout", "origin/main")
        flutter_flags = (
            ["--flutter-revision", flutter_revision] if flutter_revision else []
        )
        api.python3(
            "update dart 3p packages",
            [
                checkout_root.join("scripts", "dart", "update_3p_packages.py"),
                "--debug",
            ]
            + flutter_flags,
        )
        change = api.auto_roller.attempt_roll(
            api.auto_roller.Options(
                remote="fuchsia.googlesource.com/third_party/dart-pkg",
                commit_untracked=True,
                dry_run=dry_run,
            ),
            repo_dir=packages_root,
            commit_message=commit_msg,
        )
        # If we rolled changes, the final hash of the merged CL should
        # correspond to the remote HEAD. Otherwise, we can assume that local
        # HEAD is up to date with remote HEAD.
        current_hash = change.revision if change else api.git.get_hash()
        presentation.step_text = current_hash
    return current_hash


def GenTests(api):
    """Tests for cipd with dependents roller."""

    def properties(**kwargs):
        props = {
            "project": "integration",
            "manifests": [
                {
                    "project": "integration",
                    "manifest": "other/dependency",
                    "remote": "sso://fuchsia/integration",
                    "lock_file": "integration/jiri.lock",
                },
                {
                    "project": "integration",
                    "manifest": "fuchsia/flower",
                    "remote": "sso://fuchsia/integration",
                    "lock_file": "integration/fuchsia/jiri.lock",
                },
            ],
            "locks": [{"manifest": "a/b", "file": "a/b/jiri.lock"}],
            "orchestrator_package": "fuchsia/orchestrator",
            "orchestrator_import_in": "fuchsia/prebuilts",
            "validate_against_package": {
                "package": "flutter/fuchsia",
                "version_tag": "flutter_version",
            },
            "rolling_packages_projects": [
                {
                    "name": "dart/sdk",
                    "type": "project",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "dart_version",
                },
                {
                    "name": "skia",
                    "type": "project",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "skia_version",
                },
                {
                    "name": "flutter/fuchsia",
                    "type": "package",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "engine_version",
                },
                {
                    "import_in": "integration/fuchsia/prebuilts",
                    "name": "fuchsia/dart-sdk/${platform}",
                    "type": "package",
                    "version_tag": "dart_version",
                },
            ],
            "versions_file": "flutter/versions.json",
            "owners": ["abc@gmail.com"],
            "dry_run": True,
            "debug_symbol_packages": ["flutter/fuchsia-debug-symbols-x64"],
            "debug_symbol_gcs_buckets": ["fuchsia-debug-symbols-shortlived"],
            "dart_pkg_manifest": "manifest/dart",
        }
        props.update(kwargs)
        return api.properties(**props)

    package_no_match_test_data = api.step_data(
        "cipd describe fuchsia/orchestrator",
        api.cipd.example_describe(
            package_name="fuchsia/orchestrator",
            version="version_abc",
            test_data_tags=["git_revision:revision_abc"],
        ),
    )
    jiri_package_test_data = [
        {
            "name": "fuchsia/orchestrator",
            "path": "fuchsia/orchestrator",
            "version": "git_revision:revision_abc",
            "manifest": "manifest1",
        },
    ]

    # Version of the cipd orchestrator package is the same as the one
    # in the tree.
    yield (
        api.buildbucket_util.test("nothing_to_roll")
        + properties()
        + package_no_match_test_data
        + api.step_data("jiri package", api.jiri.package(jiri_package_test_data))
    )

    # Version of the cipd orchestrator package is different from the one
    # in the tree and ready to get rolled but flutter version is the same as the
    # one in the tree.
    package_match_test_data = api.step_data(
        "cipd describe fuchsia/orchestrator",
        api.cipd.example_describe(
            package_name="fuchsia/orchestrator",
            version="version_abc",
            test_data_tags=["git_revision:revision_jkl"],
        ),
    )
    package_versions = {
        "flutter_version": "flutter_version_xyz",
        "engine_version": "engine_version_xyz",
        "skia_version": "skia_version_xyz",
        "dart_version": "dart_version_xyz",
    }
    jiri_flutter_package_test_data = [
        {
            "name": "flutter/fuchsia",
            "path": "flutter/fuchsia",
            "version": "git_revision:flutter_version_xyz",
            "manifest": "manifest1",
        },
    ]
    yield (
        api.buildbucket_util.test("orchestrator_dep_version_same_as_tree")
        + properties()
        + api.step_data("jiri package", api.jiri.package(jiri_package_test_data))
        + api.step_data(
            "read versions", api.file.read_json(json_content=package_versions)
        )
        + package_match_test_data
        + api.step_data(
            "jiri package (2)", api.jiri.package(jiri_flutter_package_test_data)
        )
    )

    # Version of the validation cipd package is different from the one in the tree
    # and ready to get rolled along its dependencies.
    jiri_flutter_package_test_data_no_match = [
        {
            "name": "flutter/fuchsia",
            "path": "flutter/fuchsia",
            "version": "git_revision:flutter_version_xyz1",
            "manifest": "manifest1",
        },
    ]
    yield (
        api.buildbucket_util.test(
            "orchestrator_dep_version_different_from_tree_dry_run",
            status="failure",
            builder="flutter-dependents-roller",
        )
        + properties()
        + package_match_test_data
        + api.step_data("jiri package", api.jiri.package(jiri_package_test_data))
        + api.step_data(
            "read versions", api.file.read_json(json_content=package_versions)
        )
        + api.step_data(
            "jiri package (2)",
            api.jiri.package(jiri_flutter_package_test_data_no_match),
        )
        + api.auto_roller.dry_run_success(
            name="third-party dart packages.check for completion.check if done ({})"
        )
    )

    # Validate the build and upload debug symbols path.
    yield (
        api.buildbucket_util.test(
            "orchestrator_dep_version_different_from_tree",
            builder="flutter-dependents-roller",
        )
        + properties(dry_run=False)
        + package_match_test_data
        + api.step_data("jiri package", api.jiri.package(jiri_package_test_data))
        + api.step_data(
            "read versions", api.file.read_json(json_content=package_versions)
        )
        + api.step_data(
            "jiri package (2)",
            api.jiri.package(jiri_flutter_package_test_data_no_match),
        )
        + api.auto_roller.success()
        + api.auto_roller.success(
            name="third-party dart packages.check for completion.check if done (0)"
        )
    )

    # Validate the dart-sdk plugins steps is not executed when dart-sdk is not
    # being rolled.
    yield (
        api.buildbucket_util.test(
            "do_not_roll_plugins_if_dart_not_rolled",
            builder="flutter-dependents-roller",
        )
        + properties(
            dry_run=False,
            rolling_packages_projects=[
                {
                    "name": "dart/sdk",
                    "type": "project",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "dart_version",
                },
                {
                    "name": "skia",
                    "type": "project",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "skia_version",
                },
                {
                    "name": "flutter/fuchsia",
                    "type": "package",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "engine_version",
                },
            ],
        )
        + package_match_test_data
        + api.step_data("jiri package", api.jiri.package(jiri_package_test_data))
        + api.step_data(
            "read versions", api.file.read_json(json_content=package_versions)
        )
        + api.step_data(
            "jiri package (2)",
            api.jiri.package(jiri_flutter_package_test_data_no_match),
        )
        + api.auto_roller.success()
    )

    # Validate that uploading debug symbols is skipped when the flutter/fuchsia
    # package is not rolled.
    yield (
        api.buildbucket_util.test(
            "do_not_upload_debug_symbols_if_flutter_fuchsia_is_not_rolled",
            builder="flutter-dependents-roller",
        )
        + properties(
            dry_run=False,
            rolling_packages_projects=[
                {
                    "name": "dart/sdk",
                    "type": "project",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "dart_version",
                },
                {
                    "name": "skia",
                    "type": "project",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "skia_version",
                },
            ],
        )
        + package_match_test_data
        + api.step_data("jiri package", api.jiri.package(jiri_package_test_data))
        + api.step_data(
            "read versions", api.file.read_json(json_content=package_versions)
        )
        + api.step_data(
            "jiri package (2)",
            api.jiri.package(jiri_flutter_package_test_data_no_match),
        )
        + api.auto_roller.success()
    )

    # Validate that rolling_packages can use the orchestrator version_tag.
    yield (
        api.buildbucket_util.test(
            "rolling_packages_can_use_orchestrator_version_tag",
            builder="flutter-dependents-roller",
        )
        + properties(
            dry_run=False,
            rolling_packages_projects=[
                {
                    "name": "flutter/fuchsia",
                    "type": "package",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "orchestrator_version",
                },
            ],
        )
        + package_match_test_data
        + api.step_data("jiri package", api.jiri.package(jiri_package_test_data))
        + api.step_data(
            "read versions", api.file.read_json(json_content=package_versions)
        )
        + api.step_data(
            "jiri package (2)",
            api.jiri.package(jiri_flutter_package_test_data_no_match),
        )
        + api.auto_roller.success()
    )

    # Validate that test multiplier works.
    yield (
        api.buildbucket_util.test(
            "rolling_packages_multiplied", builder="flutter-dependents-roller"
        )
        + properties(
            dry_run=False,
            test_multipliers=[{"name": "test1", "total_runs": 5}],
            rolling_packages_projects=[
                {
                    "name": "flutter/fuchsia",
                    "type": "package",
                    "import_in": "fuchsia/prebuilts",
                    "version_tag": "orchestrator_version",
                },
            ],
        )
        + package_match_test_data
        + api.step_data("jiri package", api.jiri.package(jiri_package_test_data))
        + api.step_data(
            "read versions", api.file.read_json(json_content=package_versions)
        )
        + api.step_data(
            "jiri package (2)",
            api.jiri.package(jiri_flutter_package_test_data_no_match),
        )
        + api.auto_roller.success()
    )