# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for rolling flutter and dependencies.

Flutter and dependencies need to get rolled based on the version
dependencies of another fuchsia prebuilt. In this recipe we are
calling the fuchsia prebuilt dictating the dependencies "orchestrator".

The orchestrator cipd package contains a json file with the dependencies
it was built with. The following is an example of the json file:

{
 "flutter_version": "flutter_version_xyz",
 "engine_version": "engine_version_xyz",
 "skia_version": "skia_version_xyz",
 "dart_version": "dart_version_xyz"
}

This recipe checks if there is a new version of the orchestrator and
if it requires different versions of the dependencies from the ones in
the source tree. If both conditions are true then it proceeds to roll
new versions of projects and packages as described in the
rolling_packages_projects parameter and using the versions from the
orchestrator dependencies json file.

Warning: If this recipe runs in dry mode it will still roll changes into
fuchsia dart 3p repository but the fuchsia tree won't pin to the latest
version. This is safe as the fuchsia dart 3p hash will be bumped only on a
successful roll of all the other dependencies.
"""

import copy
import os

from recipe_engine.config import List
from recipe_engine.post_process import StepSuccess
from recipe_engine.recipe_api import Property

DEPS = [
    'fuchsia/jiri',
    'fuchsia/auto_roller',
    'fuchsia/buildbucket_util',
    'fuchsia/cipd_dependencies',
    'fuchsia/dart_util',
    'fuchsia/debug_symbols',
    'fuchsia/gitiles',
    'recipe_engine/archive',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/properties',
    'recipe_engine/path',
    'recipe_engine/step',
]

PROPERTIES = {
    'project':
        Property(kind=str, help='Jiri remote manifest project', default=None),
    'manifests':
        Property(
            kind=List(dict),
            help=(
                'A list of dictionaries with project, manifest and remote keys '
                'used to signal jiri wich manifests to synchronize. '
                'E.g. [{"project": "integration", "manifest": "other/dependency", '
                '      "remote": "sso://fuchsia/integration", '
                '      "lock_file": "integration/jiri.lock"}]')),
    'locks':
        Property(
            kind=List(dict),
            help=('A list of dictionaries with manifest and files keys used to '
                  'signal jiri which lock files to update. '
                  ' E.g. [{"manifest": "integration/fuchsia/flower", '
                  '        "file": "integration/fuchsia/jiri.lock"}]')),
    'orchestrator_package':
        Property(
            kind=str,
            help=('A string with the CIPD package used to validate '
                  'flutter and dependencies versions. This is a special '
                  'cipd package that contains a json file describing '
                  'the dependencies it was built with.')),
    'orchestrator_ref':
        Property(
            kind=str,
            default='latest',
            help=('A cipd ref used to get a given version of the orchestrator '
                  'package')),
    'orchestrator_import_in':
        Property(
            kind=str,
            default='',
            help=('A string with the location where the orchestrator package '
                  'should be imported into the source tree.')),
    'owners':
        Property(
            kind=List(str),
            default=(),
            help=('The owners responsible for watching this roller '
                  '(example: "username@google.com").')),
    'rolling_packages_projects':
        Property(
            kind=List(dict),
            default=(),
            help=('A list of dictionaries with rolling package/project, type,'
                  ' and location to import_in. This is used to list the '
                  'dependencies that should be updated: '
                  'E.g. [{"name": "dart", "type": "package", '
                  '       "import_in": "fuchsia/prebuilts", '
                  '       "version_tag": "dart_version"}]')),
    'versions_file':
        Property(
            kind=str,
            help=('A string with the path to the versions file inside'
                  'the orchestrator CIPD package.')),
    'validate_against_package':
        Property(
            kind=dict,
            help=('A dictionary with the package and tag for the version to use'
                  ' for validation. A single dependency is used to validate if'
                  ' a roll is needed. E.g. {"package": "flutter/fuchsia", '
                  '                          "version_tag": "flutter_version"} '
                  'will validate the source tree version of flutter/fuchsia '
                  'with the value of flutter_version from the orchestrator '
                  'json file.')),
    'dry_run':
        Property(
            kind=bool,
            default=False,
            help=('Whether to dry-run the auto-roller (CQ+1 and abandon the '
                  'change)')),
    'debug_symbol_packages':
        Property(
            kind=List(str),
            default=(),
            help=('A list of strings with the cipd packages containing debug '
                  'symbols and their associated packages.')),
    'debug_symbol_attribute':
        Property(
            kind=str,
            default='',
            help=(
                'String with the cipd attribute to validate if a cipd package '
                'contains debug symbols.')),
    'debug_symbol_import_in':
        Property(
            kind=str,
            default='',
            help=('String with the path where debug symbols will be imported '
                  'in the source tree.')),
    'debug_symbol_gcs_buckets':
        Property(
            kind=List(str),
            default=(),
            help=('List of strings with the gcs buckets where the debug '
                  'symbols will be uploaded to.')),
    'debug_symbol_remote':
        Property(
            kind=str,
            default='',
            help='String with the url of the debug symbol remote repository')
}

COMMIT_MESSAGE = """[roll] Roll {roller} packages and projects:

{packages}

Test: CQ
CQ-Do-Not-Cancel-Tryjobs: true"""


def RunSteps(api, project, manifests, locks, orchestrator_ref, owners,
             versions_file, rolling_packages_projects, orchestrator_package,
             orchestrator_import_in, validate_against_package, dry_run,
             debug_symbol_packages, debug_symbol_attribute,
             debug_symbol_gcs_buckets, debug_symbol_import_in,
             debug_symbol_remote):
  """Run the recipe steps."""
  with api.context(infra_steps=True):
    if owners:
      with api.step.nest('owners') as presentation:
        presentation.step_summary_text = ', '.join(owners)

    # Import manifests
    with api.step.nest('jiri-manifests') as presentation:
      api.jiri.init(use_lock_file=True)
      presentation.logs['manifests'] = str(manifests)
      for manifest_config in manifests:
        api.jiri.import_manifest(
            manifest=manifest_config['manifest'],
            remote=manifest_config['remote'],
            name=manifest_config['project'])
      api.jiri.update(run_hooks=False)
      api.jiri.run_hooks()

  deps_info = api.jiri.package([orchestrator_package])
  cipd_description = api.cipd.describe(orchestrator_package, orchestrator_ref)
  orchestrator_current_version = deps_info.json.output[0]['version']
  # Is there a new version to roll?
  orchestrator_expected_version = cipd_description.tags[0].tag
  if orchestrator_expected_version == orchestrator_current_version:
    msg = '%s in tree version is [%s] and latest from cipd is [%s]' % (
        orchestrator_package, orchestrator_current_version,
        orchestrator_expected_version)
    api.step('manifest up-to-date; nothing to roll',
             None).presentation.step_text = msg
    return

  # Read main package versions file.
  orchestrator_version_dict = api.cipd_dependencies.get_dependencies(
      'read versions', orchestrator_package, cipd_description.pin.instance_id,
      versions_file)

  # Use version of the validation package to decide whether a roll is needed or
  # not.
  validation_info = api.jiri.package([validate_against_package['package']])
  local_version = validation_info.json.output[0]['version']
  expected_version = 'git_revision:%s' % orchestrator_version_dict[
      validate_against_package['version_tag']]
  if local_version == expected_version:
    msg = 'Expected version [%s] == local fuchsia tree version [%s]' % (
        expected_version, local_version)
    api.step('validation_summary', cmd=None).presentation.step_text = msg
    return
  # Start rolling orchestrator
  rolling_packages_projects.append({
      'type': 'package',
      'name': orchestrator_package,
      'version_tag': orchestrator_package,
      'import_in': orchestrator_import_in
  })
  orchestrator_version_dict[
      orchestrator_package] = orchestrator_expected_version
  # Roll all the dependencies
  package_msgs = []
  package_msg_template = '%s %s rolled from %s to %s'
  with api.step.nest('edit jiri manifests'):
    for package_project in rolling_packages_projects:
      package_msg = ''
      if package_project['type'] == 'package':
        if package_project['name'] == orchestrator_package:
          expected_version = orchestrator_version_dict[
              package_project['version_tag']]
        else:
          expected_version = 'git_revision:%s' % orchestrator_version_dict[
              package_project['version_tag']]
        changes = api.jiri.edit_manifest(
            package_project['import_in'],
            packages=[(package_project['name'], expected_version)],
            name=package_project['name'])
        # Sometimes only a subset of packages are updated. In those cases
        # the changes list is empty.
        if changes['packages']:
          old_version = changes['packages'][0]['old_version']
          package_msg = package_msg_template % (
              'package', package_project['name'], old_version, expected_version)
      else:
        expected_version = orchestrator_version_dict[
            package_project['version_tag']]
        # We need to save dart/sdk version to update the dart
        # plugins version later in this recipe.
        if package_project['name'] == 'dart/sdk':
          dart_sdk_revision = expected_version
        changes = api.jiri.edit_manifest(
            package_project['import_in'],
            name=package_project['name'],
            projects=[(package_project['name'], expected_version)])
        # Sometimes only a subset of projects are updated. In those cases
        # the changes list is empty.
        if changes['projects']:
          old_version = changes['projects'][0]['old_revision']
          package_msg = package_msg_template % (
              'project', package_project['name'], old_version, expected_version)
      if package_msg:
        package_msgs.append(package_msg)

  # Roll dart internal plugins
  sandbox_dir = api.path.mkdtemp('sandbox-flutter-dart')
  dart_path = sandbox_dir.join('dart')
  with api.step.nest('dart sdk checkout'):
    api.dart_util.checkout(path=dart_path, revision=dart_sdk_revision)

  with api.step.nest('dart third-party packages'):
    checkout_root = api.path['start_dir'].join('integration')
    api.dart_util.update_pkg_manifest(
        path=dart_path, checkout_root=checkout_root)

  # Update fuchsia's third-party dart packages, not to be confused with
  # the previous updating of dart's own third-party packages.
  manifest_repo = api.path['start_dir'].join('integration')
  manifest_path = manifest_repo.join('fuchsia', 'topaz', 'topaz')
  with api.step.nest('third-party dart packages'):
    checkout_root = api.path['start_dir']
    rolled_hash = api.dart_util.update_3p_packages(checkout_root=checkout_root)
    changes = api.jiri.edit_manifest(
        'integration/fuchsia/topaz/dart',
        name='third_party/dart-pkg',
        projects=[('third_party/dart-pkg', rolled_hash)])

  # Update lock files
  with api.step.nest('update jiri lock files'):
    for lock in locks:
      api.jiri.resolve(
          local_manifest=True,
          output=lock['file'],
          manifests=[lock['manifest']])

  roller_name = api.buildbucket.builder_name
  if roller_name.endswith('-roller'):
    roller_name = api.buildbucket.builder_name[:7]
  message = COMMIT_MESSAGE.format(
      roller=roller_name,
      packages='\n\n'.join(package_msgs),
      builder=api.buildbucket.builder_name,
      build_id=api.buildbucket_util.id,
  )

  # All the changes are happening inside one of the git cloned projects.
  # The "project" property points to the location of the project containing
  # the changes and we need to move to that directory for the roller to pick
  # up the changes.
  project_dir = api.path['start_dir'].join(*project.split('/'))
  rolled = False
  with api.context(cwd=project_dir):
    rolled = api.auto_roller.attempt_roll(
        gerrit_project=project,
        repo_dir=project_dir,
        commit_message=message,
        dry_run=dry_run,
    )

  # Push debug symbols.
  # If running on dry_run mode skip building and pushing debug symbols
  # because the correct dependencies won't be available in the tree.
  if rolled and debug_symbol_gcs_buckets and not dry_run:
    checkout_root = api.path['start_dir']
    api.debug_symbols.fetch_and_upload(
        project=project,
        checkout_root=checkout_root,
        import_in=debug_symbol_import_in,
        remote=debug_symbol_remote,
        project_dir=project_dir,
        packages=debug_symbol_packages,
        debug_symbol_attribute=debug_symbol_attribute,
        debug_symbol_gcs_buckets=debug_symbol_gcs_buckets)


def GenTests(api):
  """Tests for cipd with dependents roller."""
  properties = {
      'project': 'integration',
      'manifests': [{
          'project': 'integration',
          'manifest': 'other/dependency',
          'remote': 'sso://fuchsia/integration',
          'lock_file': 'integration/jiri.lock'
      }, {
          'project': 'integration',
          'manifest': 'fuchsia/flower',
          'remote': 'sso://fuchsia/integration',
          'lock_file': 'integration/fuchsia/jiri.lock'
      }],
      'locks': [{
          'manifest': 'a/b',
          'file': 'a/b/jiri.lock'
      }],
      'orchestrator_package': 'fuchsia/orchestrator',
      'orchestrator_import_in': 'fuchsia/prebuilts',
      'validate_against_package': {
          'package': 'flutter/fuchsia',
          'version_tag': 'flutter_version'
      },
      'rolling_packages_projects': [
          {
              'name': 'dart/sdk',
              'type': 'project',
              'import_in': 'fuchsia/prebuilts',
              'version_tag': 'dart_version'
          },
          {
              'name': 'skia',
              'type': 'project',
              'import_in': 'fuchsia/prebuilts',
              'version_tag': 'skia_version'
          },
          {
              'name': 'flutter',
              'type': 'package',
              'import_in': 'fuchsia/prebuilts',
              'version_tag': 'engine_version'
          },
      ],
      'versions_file': 'flutter/versions.json',
      'owners': ['abc@gmail.com'],
      'dry_run': True,
      'debug_symbol_packages': ['flutter/fuchsia-debug-symbols-x64',],
      'debug_symbol_attribute': 'debug-symbols',
      'debug_symbol_gcs_buckets': ['fuchsia-debug-symbols-shortlived'],
      'debug_symbol_import_in': 'integration/fuchsia/prebuilts',
      'debug_symbol_remote': 'sso://fuchsia/integration'
  }

  package_no_match_test_data = api.step_data(
      'cipd describe fuchsia/orchestrator',
      api.cipd.example_describe(
          package_name='fuchsia/orchestrator',
          version='version_abc',
          test_data_tags=['git_revision:revision_abc']))
  jiri_package_test_data = [
      {
          'name': 'fuchsia/orchestrator',
          'path': 'fuchsia/orchestrator',
          'version': 'git_revision:revision_abc',
          'manifest': 'manifest1'
      },
  ]
  default_properties = api.properties(**properties)
  # Version of the cipd orchestrator package is the same as the one
  # in the tree.
  yield (
      api.test('nothing_to_roll') + default_properties +
      package_no_match_test_data +
      api.step_data('jiri package', api.jiri.package(jiri_package_test_data)))

  # Version of the cipd orchestrator package is different from the one
  # in the tree and ready to get rolled but flutter version is the same as the
  # one in the tree.
  package_match_test_data = api.step_data(
      'cipd describe fuchsia/orchestrator',
      api.cipd.example_describe(
          package_name='fuchsia/orchestrator',
          version='version_abc',
          test_data_tags=['git_revision:revision_jkl']))
  package_versions = {
      'flutter_version': 'flutter_version_xyz',
      'engine_version': 'engine_version_xyz',
      'skia_version': 'skia_version_xyz',
      'dart_version': 'dart_version_xyz'
  }
  jiri_flutter_package_test_data = [
      {
          'name': 'flutter/fuchsia',
          'path': 'flutter/fuchsia',
          'version': 'git_revision:flutter_version_xyz',
          'manifest': 'manifest1'
      },
  ]
  yield (api.test('orchestrator_dep_version_same_as_tree') +
         default_properties + package_match_test_data + api.step_data(
             'jiri package', api.jiri.package(jiri_package_test_data)) +
         api.step_data('read versions',
                       api.file.read_json(json_content=package_versions)) +
         package_match_test_data +
         api.step_data('jiri package (2)',
                       api.jiri.package(jiri_flutter_package_test_data)) +
         api.post_process(StepSuccess, 'validation_summary'))

  # Version of the validation cipd package is different from the one in the tree
  # and ready to get rolled along its dependencies.
  jiri_flutter_package_test_data_no_match = [
      {
          'name': 'flutter/fuchsia',
          'path': 'flutter/fuchsia',
          'version': 'git_revision:flutter_version_xyz1',
          'manifest': 'manifest1'
      },
  ]
  yield (
      api.test('orchestrator_dep_version_different_from_tree_dry_run') +
      default_properties + package_match_test_data +
      api.step_data('jiri package', api.jiri.package(jiri_package_test_data)) +
      api.step_data('read versions',
                    api.file.read_json(json_content=package_versions)) +
      api.step_data('jiri package (2)',
                    api.jiri.package(jiri_flutter_package_test_data_no_match)) +
      api.auto_roller.success_step_data() + api.buildbucket.build(
          api.buildbucket.ci_build_message(builder='flutter-dependents-roller'))
      + api.auto_roller.success_step_data(
          name='third-party dart packages.check for completion.check if done (0)'
      ))

  # Validate the build and upload debug symbols path.
  props = copy.deepcopy(properties)
  props['dry_run'] = False
  no_dry_run_props = api.properties(**props)
  yield (
      api.test('orchestrator_dep_version_different_from_tree') +
      no_dry_run_props + package_match_test_data +
      api.step_data('jiri package', api.jiri.package(jiri_package_test_data)) +
      api.step_data('read versions',
                    api.file.read_json(json_content=package_versions)) +
      api.step_data('jiri package (2)',
                    api.jiri.package(jiri_flutter_package_test_data_no_match)) +
      api.auto_roller.success_step_data() + api.buildbucket.build(
          api.buildbucket.ci_build_message(builder='flutter-dependents-roller'))
      + api.auto_roller.success_step_data(
          name='third-party dart packages.check for completion.check if done (0)'
      ) + api.jiri.read_manifest_element(
          api,
          'fuchsia/prebuilts',
          'package',
          'flutter/fuchsia-debug-symbols-x64',
          test_output={
              'path': 'prebuilt/build_ids/arm64/flutter',
              'attributes': 'debug-symbols',
          }))