# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for automatically updating Flutter and Dart."""

from collections import OrderedDict

import os
import re

from recipe_engine.config import List
from recipe_engine.recipe_api import Property

DEPS = [
    'fuchsia/auto_roller',
    'fuchsia/build',
    'fuchsia/buildbucket_util',
    'fuchsia/git',
    'fuchsia/gitiles',
    'fuchsia/jiri',
    'fuchsia/upload_debug_symbols',
    'recipe_engine/archive',
    'recipe_engine/buildbucket',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/raw_io',
    'recipe_engine/step',
]

PROPERTIES = {
    'manifest':
        Property(kind=str, help='Jiri manifest to use', default='topaz/topaz'),
    'remote':
        Property(
            kind=str,
            help='Remote manifest repository',
            default='https://fuchsia.googlesource.com/integration'),
    'flutter_manifest':
        Property(
            kind=str, help='Jiri manifest for flutter',
            default='topaz/flutter'),
    'dart_manifest':
        Property(kind=str, help='Jiri manifest for dart', default='topaz/dart'),
    'dart_third_party_pkg_manifest':
        Property(
            kind=str,
            help='Jiri manifest for dart_third_party_pkg',
            default='topaz/dart_third_party_pkg'),
    'revision':
        Property(kind=str, help='flutter/flutter revision', default=''),
    'prebuilt_manifest':
        Property(
            kind=str,
            help='Jiri manifest for prebuilt packages',
            default='fuchsia/prebuilts'),
    'lockfiles':
        Property(
            kind=List(str),
            default=[],
            help=('The list of lockfiles to update in "${manifest}=${lockfile}"'
                  ' format')),
    # TODO: delete this property after we have full lockfile support in
    # integration repo
    'enforce_locks':
        Property(
            kind=bool,
            default=False,
            help='Whether to enforce locks from lockfiles'),
    'debug_symbol_attribute':
        Property(
            kind=str,
            default='debug-symbols',
            help='Jiri attribute to match debug symbol packages'),
    'debug_symbol_gcs_buckets':
        Property(
            kind=List(str),
            default=(),
            help='GCS buckets to upload debug symbols upon successful roll'),
}

FLUTTER_NAME = 'external/github.com/flutter/flutter'
ENGINE_REMOTE = 'https://chromium.googlesource.com/external/github.com/flutter/engine'
DART_SDK_NAME = 'dart/sdk'
DART_SDK_PKG_NAME = 'fuchsia/dart-sdk/${platform}'
DEBUG_SYMBOL_PACKAGES = [
    'flutter/fuchsia-debug-symbols-x64', 'flutter/fuchsia-debug-symbols-arm64'
]
FLUTTER_CIPD_PACKAGES = ['flutter/fuchsia', 'flutter/sky_engine'
                        ] + DEBUG_SYMBOL_PACKAGES
THIRD_PARTY_DART_PKG_NAME = 'third_party/dart-pkg'
THIRD_PARTY_DART_PKG_PATH = ['third_party', 'dart-pkg', 'pub']

COMMIT_MESSAGE = """\
[roll] Update {deps}

{logs}

Cq-Cl-Tag: roller-builder:{builder}
Cq-Cl-Tag: roller-bid:{build_id}
CQ-Do-Not-Cancel-Tryjobs: true"""

LOCAL_IMPORT_FORMAT = """\
<manifest>
  <imports>
    <localimport file="{local_manifest}"/>
  </imports>
</manifest>
"""

THIRD_PARTY_DART_PKG_COMMIT_MSG = """\
[roll] Update third-party dart packages

Updated:
{updated}
"""


def extract_dependency_version_from_deps(api, deps_path, dependency):
  """Extracts the dependency version from a flutter/engine's DEPS file.

  Args:
    api (RecipeApi): Recipe API object.
    deps_path (Path): A path to a DEPS file.
    manifest_path (Path): A path to the Jiri manifest to overwrite.
    dependency (str): The name of a dependency to extract from the DEPS file.
  """
  dependency_revision = '%s_revision' % dependency
  contents = api.file.read_text(
      name='read DEPS file for %s' % dependency,
      source=deps_path,
      test_data="'%s': 'abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde'" %
      dependency_revision,
  )
  m = re.search(r"'%s':\s*'(?P<revision>[0-9a-f]{40})'" % dependency_revision,
                contents)
  if not m:
    raise api.step.InfraFailure('failed to find %s in DEPS' %
                                dependency_revision)
  return m.group('revision')


def update_pkg_manifest(api, dart_path, manifest_path):
  """Overwrites a dart third party package manifest.

  Args:
    api (RecipeApi): Recipe API object.
    dart_path (Path): A path to the dart/sdk repository.
    manifest_path (Path): A path to the Jiri manifest to overwrite.
  """
  api.python(
      name='update %s' % api.path.basename(manifest_path),
      script=dart_path.join('tools', 'create_pkg_manifest.py'),
      args=['-d', dart_path.join('DEPS'), '-o', manifest_path],
  )


def roll_changes(api, deps):
  """Rolls manifest changes in a git repository.

  Args:
    deps (dict[str]str): A map of dependencies that were updated to
    a log string, summarizing the update.
  """
  # Generate the commit message.
  commit_message = COMMIT_MESSAGE.format(
      deps=', '.join(deps.keys()),
      logs='\n'.join(deps.itervalues()),
      builder=api.buildbucket.builder_name,
      build_id=api.buildbucket_util.id,
  )

  # Land the changes.
  return api.auto_roller.attempt_roll(
      gerrit_project='integration',
      repo_dir=api.path['start_dir'].join('integration'),
      commit_message=commit_message,
  )


def update_3p_dart_packages(api, jiri_manifest, local_manifest):
  """Updates fuchsia's third-party packages.

  This is a very stateful operation: in particular, it overwrites a
  .jiri_manifest, temporarily stashes the existing manifest changes, updates per
  the changes made in a local manifest repo, and pushes a change from
  //third_party/dart-pkg directly to its origin/master.

  Args:
    jiri_manifest (Path): The path to a .jiri_manifest.
    local_manifest (str): The relative path to a local manifest from the jiri
      root.
  """
  api.file.write_raw(
      'overwrite jiri manifest',
      jiri_manifest,
      LOCAL_IMPORT_FORMAT.format(local_manifest=local_manifest),
  )
  # Stash the manifest changes while we are updating to prevent overwriting.
  manifest_repo = api.path['start_dir'].join('integration')
  with api.context(cwd=manifest_repo):
    api.git('stash', 'push', '--include-untracked')
    api.jiri.update(fetch_packages=True, gc=True)
    api.git('stash', 'pop')

  third_party_dart_pkg_repo = api.path['start_dir'].join(
      *THIRD_PARTY_DART_PKG_PATH)
  with api.context(cwd=third_party_dart_pkg_repo):
    # Make sure third_party/dart-pkg is at origin/master before running the
    # update script to catch any manual commits that extend past the revision at
    # integration's HEAD.
    api.step('fetch', ['git', 'fetch', 'origin'])
    api.step('checkout', ['git', 'checkout', 'origin/master'])

    api.python(
        'update',
        api.path['start_dir'].join('scripts', 'dart', 'update_3p_packages.py'),
        args=['--debug'],
    )

    changed_files = api.git(
        'ls-files',
        'modified',
        '--deleted',
        '--others',
        '--exclude-standard',
        stdout=api.raw_io.output(),
    ).stdout

    commit_msg = THIRD_PARTY_DART_PKG_COMMIT_MSG.format(updated=changed_files)
    commit_step = api.git.commit(
        message=commit_msg,
        all_files=True,
        ok_ret='any',
    )
    commit_step.presentation.logs['commit message'] = commit_msg.splitlines()

    # Since we fetch first and since we are now at or ahead of origin/master,
    # git push is at worst a zero-returning no-op, so cannot be a failing step.
    api.step('fetch', ['git', 'fetch', 'origin'])
    api.git.push(ref='HEAD:master')

    revision = api.git.get_hash()
    api.step.active_result.presentation.logs['revision'] = [revision]
    return revision


def update_deps(api, updated_deps, manifest_path, flutter_revision,
                flutter_manifest_path, dart_manifest_path,
                dart_third_party_pkg_manifest_path, prebuilt_manifest_path):
  """Updates dart- and flutter-related dependencies to be reflected in the roll.

    This method makes local changes to a checked out manifest repo.

    Args:
      updated_deps (OrderedDict): A dictionary mapping project/package name to
        a formatted log of the corresponding changes.
      manifest_path (Path): The path to the main checked out manifest.
      flutter_revision (str): The revision at which to checkout out
        flutter/flutter.
      flutter_manifest_path (Path): The path to the flutter manifest.
      dart_manifest_path (Path): The path to the dart manifest.
      dart_third_party_pkg_manifest_path (Path): The path to the manifest of
        dart's third-party dependencies.
      prebuilt_manifest_path (Path): The path to the prebuilt manifest.
    """
  # Set up a temporary sandbox directory to do the required manipulations to
  # roll flutter, flutter engine, and dart.
  sandbox_dir = api.path.mkdtemp('sandbox-flutter-dart')
  # Attempt to update the manifest with a new flutter revision.
  with api.step.nest('flutter/flutter'):
    flutter_remote = api.auto_roller.update_manifest_project(
        manifest=flutter_manifest_path,
        project_name=FLUTTER_NAME,
        revision=flutter_revision,
        updated_deps=updated_deps,
    )
    if not updated_deps:
      return

    # Get the flutter/flutter dependency on flutter/engine.
    flutter_path = sandbox_dir.join('flutter')
    api.git.checkout(
        url=flutter_remote,
        path=flutter_path,
        ref=flutter_revision,
    )
    engine_revision = api.file.read_text(
        name='read flutter engine version',
        source=flutter_path.join('bin', 'internal', 'engine.version'),
        test_data='xyz000',
    ).strip()

  # Attempt to update the manifest with a new engine revision.
  with api.step.nest('flutter/engine'):
    # Get the flutter/engine dependency on Dart & Skia.
    engine_path = sandbox_dir.join('engine')

    api.git.checkout(
        url=ENGINE_REMOTE,
        path=engine_path,
        ref=engine_revision,
    )

    # Update flutter prebuilts
    for prebuilt in FLUTTER_CIPD_PACKAGES:
      api.auto_roller.update_manifest_package(
          manifest=prebuilt_manifest_path,
          package_name=prebuilt,
          version='git_revision:%s' % engine_revision,
          updated_deps=updated_deps,
      )

  with api.step.nest('dart sdk'):
    dart_revision = extract_dependency_version_from_deps(
        api, engine_path.join('DEPS'), 'dart')
    dart_remote = api.auto_roller.update_manifest_project(
        manifest=dart_manifest_path,
        project_name=DART_SDK_NAME,
        revision=dart_revision,
        updated_deps=updated_deps,
    )

    # Update prebuilt for dart/sdk
    api.jiri.edit_manifest(
        manifest=prebuilt_manifest_path,
        packages=[(DART_SDK_PKG_NAME, 'git_revision:' + dart_revision)])

    # Get dart/sdk.
    dart_path = sandbox_dir.join('dart')
    api.git.checkout(
        url=dart_remote,
        path=dart_path,
        ref=dart_revision,
    )

  # Update the package manifests.
  with api.step.nest('dart third-party packages'):
    update_pkg_manifest(
        api,
        dart_path=dart_path,
        manifest_path=dart_third_party_pkg_manifest_path,
    )

  # Update fuchsia's third-party dart packages, not to be confused with
  # the previous updating of dart's own third-party packages.
  with api.step.nest('third-party dart packages'):
    third_party_dart_pkg_revision = update_3p_dart_packages(
        api=api,
        jiri_manifest=api.path['start_dir'].join('.jiri_manifest'),
        local_manifest=os.path.relpath(
            str(manifest_path),
            str(api.path['start_dir']),
        ),
    )
    api.auto_roller.update_manifest_project(
        manifest=dart_manifest_path,
        project_name=THIRD_PARTY_DART_PKG_NAME,
        revision=third_party_dart_pkg_revision,
        updated_deps=updated_deps,
    )


def fetch_and_upload_debug_symbols(api, project, import_in, remote, project_dir,
                                   packages, debug_symbol_attribute,
                                   debug_symbol_gcs_buckets):
  """
  Fetch debug symbol archives, unpack them, and upload debug symbols.

  Args:
    project (str): Jiri remote manifest project.
    import_in (str): Path to the edited manifest relative to $project
      containing debug symbol packages.
    remote (str): Remote manifest repository.
    project_dir (Path): Project root path of $import_in.
    packages (seq(str)): The list of CIPD packages updated in $import_in.
    debug_symbol_attribute (str): Jiri attribute to match debug symbol packages.
    debug_symbol_gcs_buckets (seq(str)): GCS buckets to upload debug symbols to.
  """
  with api.context(infra_steps=True):
    api.jiri.init(
        use_lock_file=True,
        attributes=(debug_symbol_attribute,),
    )
    # Fetch debug symbol packages using locally edited manifest.
    api.jiri.import_manifest(
        manifest=import_in,
        remote=remote,
        name=project,
    )
    api.jiri.fetch_packages(local_manifest=True)

  with api.step.nest('build'):
    gn_results = api.build.gen(
        checkout_root=api.path['start_dir'],
        fuchsia_build_dir=api.path['start_dir'].join('out', 'default'),
        target='x64',
        build_type='debug',
        product='products/bringup.gni',
        # //bundles:infratools is necessary to build upload_debug_symbols.
        packages=["//bundles:infratools"],
    )

    upload_debug_symbols_target = os.path.relpath(
        str(gn_results.tool('upload_debug_symbols')),
        str(gn_results.fuchsia_build_dir),
    )
    api.build.ninja(
        gn_results=gn_results,
        build_zircon=False,
        targets=[upload_debug_symbols_target],
    )

  build_id_dirs = []
  for package in packages:
    # Find archives for each debug symbol package.
    with api.context(cwd=project_dir):
      package_def = api.jiri.read_manifest_element(
          manifest=import_in,
          element_type='package',
          element_name=package,
      )
    # Skip non debug symbol packages.
    if debug_symbol_attribute not in package_def.get('attributes', ''):
      continue

    package_path = api.path['start_dir'].join(package_def['path'])
    archives = api.file.glob_paths(
        name='find archives for %s' % package,
        source=package_path,
        pattern='**/*.tar.bz2',
        test_data=(package_path.join('symbols.tar.bz2'),),
    )

    # Unpack archives into .build-id dirs.
    for archive in archives:
      # Extract API requires a unique, non-existent directory.
      archive_basename = os.path.basename(api.path.abspath(archive))
      output_dir = api.path['start_dir'].join(package, archive_basename)
      api.archive.extract(
          step_name='extract %s' % archive,
          archive_file=archive,
          output=output_dir,
      )
      build_id_dirs.append(output_dir)

  for debug_symbol_gcs_bucket in debug_symbol_gcs_buckets:
    api.upload_debug_symbols(
        step_name='upload debug symbols',
        upload_debug_symbols_path=gn_results.tool('upload_debug_symbols'),
        bucket=debug_symbol_gcs_bucket,
        build_id_dirs=build_id_dirs,
    )


def RunSteps(api, manifest, remote, revision, dart_manifest, flutter_manifest,
             dart_third_party_pkg_manifest, prebuilt_manifest, lockfiles,
             enforce_locks, debug_symbol_attribute, debug_symbol_gcs_buckets):
  with api.context(infra_steps=True):
    api.jiri.init(use_lock_file=enforce_locks)
    api.jiri.import_manifest(
        manifest=manifest,
        remote=remote,
        name='integration',
    )
    api.jiri.update(run_hooks=False)

    flutter_revision = revision or api.buildbucket.build.input.gitiles_commit.id

    manifest_repo = api.path['start_dir'].join('integration')
    manifest_path = manifest_repo.join(manifest)
    flutter_manifest_path = manifest_repo.join(*flutter_manifest.split('/'))
    dart_manifest_path = manifest_repo.join(*dart_manifest.split('/'))
    dart_third_party_pkg_manifest_path = manifest_repo.join(
        *dart_third_party_pkg_manifest.split('/'))
    prebuilt_manifest_path = manifest_repo.join(*prebuilt_manifest.split('/'))

    updated_deps = OrderedDict()
    update_deps(api, updated_deps, manifest_path, flutter_revision,
                flutter_manifest_path, dart_manifest_path,
                dart_third_party_pkg_manifest_path, prebuilt_manifest_path)
    # Update lockfiles
    with api.context(cwd=manifest_repo):
      for lock_entry in lockfiles:
        fields = lock_entry.split('=')
        manifest = fields[0]
        lock = fields[1]
        api.jiri.resolve(local_manifest=True, output=lock, manifests=[manifest])
    if updated_deps:
      rolled = roll_changes(api, updated_deps)

      # If debug symbol packages were rolled, upload debug symbols.
      # TODO(fxb/37432): Upload debug symbols with artifactory.
      packages_updated = set(DEBUG_SYMBOL_PACKAGES).issubset(updated_deps)
      if rolled and packages_updated and debug_symbol_gcs_buckets:
        with api.step.nest('fetch and upload debug symbols'):
          fetch_and_upload_debug_symbols(
              api=api,
              project='integration',
              import_in=prebuilt_manifest,
              remote=remote,
              project_dir=manifest_repo,
              packages=FLUTTER_CIPD_PACKAGES,
              debug_symbol_attribute=debug_symbol_attribute,
              debug_symbol_gcs_buckets=debug_symbol_gcs_buckets,
          )


def GenTests(api):
  project_names = {
      'flutter/flutter': FLUTTER_NAME,
      'dart sdk': DART_SDK_NAME,
      'third-party dart packages': THIRD_PARTY_DART_PKG_NAME,
  }
  noop_edit = lambda name: api.step_data(
      '%s.jiri edit %s' % (name, project_names[name]),
      api.json.output({'projects': []}),
  )
  noop_package_edit = lambda name: api.step_data(
      'flutter/engine.jiri edit %s' % name,
      api.json.output({'packages': []}),
  )

  def log_data(name):
    return api.gitiles.log('%s.log %s' % (name, project_names[name]), 'A')

  flutter_check_data = api.jiri.read_manifest_element(
      api=api,
      manifest='topaz/flutter',
      element_name=FLUTTER_NAME,
      element_type='project',
      test_output={
          'remote': 'https://fuchsia.googlesource.com/third_party/flutter',
      },
      nesting='flutter/flutter')
  flutter_log_data = log_data('flutter/flutter')

  dart_sdk_check_data = api.jiri.read_manifest_element(
      api=api,
      manifest='topaz/dart',
      element_name=DART_SDK_NAME,
      element_type='project',
      test_output={
          'remote': 'https://fuchsia.googlesource.com/third_party/dart',
      },
      nesting='dart sdk')
  dart_sdk_log_data = log_data('dart sdk')

  third_party_dart_pkg_check_data = api.jiri.read_manifest_element(
      api=api,
      manifest='topaz/dart',
      element_name=THIRD_PARTY_DART_PKG_NAME,
      element_type='project',
      test_output={
          'remote': 'https://fuchsia.googlesource.com/third_party/dart-pkg',
      },
      nesting='third-party dart packages')
  third_party_dart_pkg_log_data = log_data('third-party dart packages')

  yield (api.test('noop roll') + api.properties(revision='abc123') +
         flutter_check_data + noop_edit('flutter/flutter'))

  yield (api.test('noop roll with lockfile') + api.properties(
      revision='abc123', lockfiles=['manifest/topaz=jiri.lock']) +
         flutter_check_data + noop_edit('flutter/flutter'))

  yield (api.test('flutter/flutter only') + api.properties(
      revision='abc123', debug_symbol_gcs_buckets=('foo-bucket', 'bar-bucket'))
         + flutter_check_data + flutter_log_data + dart_sdk_check_data +
         third_party_dart_pkg_check_data + noop_edit('dart sdk') +
         noop_edit('third-party dart packages') +
         noop_package_edit('flutter/fuchsia-debug-symbols-x64') +
         api.auto_roller.success_step_data())

  yield (api.test('cannot find dart version') +
         api.properties(revision='abc123') + flutter_check_data +
         flutter_log_data + api.step_data('dart sdk.read DEPS file for dart',
                                          api.raw_io.output_text('stuff')))

  yield (api.test('flutter/flutter and dart') +
         api.properties(revision='abc123') + flutter_check_data +
         flutter_log_data + dart_sdk_check_data + dart_sdk_log_data +
         third_party_dart_pkg_check_data + third_party_dart_pkg_log_data +
         api.auto_roller.success_step_data())

  yield (api.test('no revision') + flutter_check_data + flutter_log_data +
         dart_sdk_check_data + dart_sdk_log_data +
         third_party_dart_pkg_check_data + third_party_dart_pkg_log_data +
         api.auto_roller.success_step_data() +
         api.buildbucket.ci_build(revision='321abc'))

  yield (api.test('with debug symbols') + api.properties(
      revision='abc123', debug_symbol_gcs_buckets=('foo-bucket', 'bar-bucket'))
         + flutter_check_data + flutter_log_data + dart_sdk_check_data +
         third_party_dart_pkg_check_data + noop_edit('dart sdk') +
         noop_edit('third-party dart packages') +
         api.auto_roller.success_step_data() + api.jiri.read_manifest_element(
             api,
             'fuchsia/prebuilts',
             'package',
             'flutter/fuchsia',
             test_output={
                 'path': 'prebuilt/flutter',
             },
             nesting='fetch and upload debug symbols') +
         api.jiri.read_manifest_element(
             api,
             'fuchsia/prebuilts',
             'package',
             'flutter/fuchsia-debug-symbols-arm64',
             test_output={
                 'path': 'prebuilt/build_ids/arm64/flutter',
                 'attributes': 'debug-symbols,debug-symbols-arm64',
             },
             nesting='fetch and upload debug symbols') +
         api.jiri.read_manifest_element(
             api,
             'fuchsia/prebuilts',
             'package',
             'flutter/fuchsia-debug-symbols-x64',
             test_output={
                 'path': 'prebuilt/build_ids/x64/flutter',
                 'attributes': 'debug-symbols,debug-symbols-x64',
             },
             nesting='fetch and upload debug symbols') +
         api.jiri.read_manifest_element(
             api,
             'fuchsia/prebuilts',
             'package',
             'flutter/sky_engine',
             test_output={
                 'path': 'prebuilt/sky_engine',
             },
             nesting='fetch and upload debug symbols'))