# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for rolling CIPD prebuilts into Fuchsia."""

import os
import re

from recipe_engine.config import List
from recipe_engine.recipe_api import Property

DEPS = [
    'fuchsia/auto_roller',
    'fuchsia/build',
    'fuchsia/buildbucket_util',
    'fuchsia/jiri',
    'fuchsia/upload_debug_symbols',
    'recipe_engine/archive',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/step',
]

PROPERTIES = {
    'project':
        Property(kind=str, help='Jiri remote manifest project', default=None),
    'manifest':
        Property(kind=str, help='Jiri manifest to use'),
    'remote':
        Property(kind=str, help='Remote manifest repository'),
    'import_in':
        Property(
            kind=str, help='Path to the manifest to edit relative to $project'),
    'packages':
        Property(
            kind=List(str),
            help='The list of CIPD packages to update in $import_in'),
    'debug_symbol_attribute':
        Property(
            kind=str,
            default='debug-symbols',
            help='Jiri attribute to match debug symbol packages'),
    'debug_symbol_gcs_buckets':
        Property(
            kind=List(str),
            default=(),
            help='GCS buckets to upload debug symbols upon successful roll'),
    'lockfiles':
        Property(
            kind=List(str),
            default=(),
            help='The list of lockfiles to update in "${manifest}=${lockfile}" format'
        ),
    'dry_run':
        Property(
            kind=bool,
            default=False,
            help='Whether to dry-run the auto-roller (CQ+1 and abandon the change)'
        ),
    'tag':
        Property(
            kind=str,
            default='version',
            help='A CIPD tag common to all $packages where a common version can be extracted'
        ),
    'ref':
        Property(
            kind=str,
            default='latest',
            help='A common CIPD ref to resolve when rolling a set of packages'),
    'owners':
        Property(
            kind=List(str),
            default=(),
            help=('The owners responsible for watching this roller '
                  '(example: "username@google.com").')),
}

COMMIT_MESSAGE = """[roll] Roll {roller} CIPD packages:

{packages}

From: {old_version}
To: {version}

Cq-Cl-Tag: roller-builder:{builder}
Cq-Cl-Tag: roller-bid:{build_id}
CQ-Do-Not-Cancel-Tryjobs: true"""

CIPD_URL = 'https://chrome-infra-packages.appspot.com/p/{package}/+/{version}'


def get_tags_from_cipd(api, cipd_pkg, tag, ref):
  cipd_tags = api.cipd.describe(cipd_pkg, ref).tags
  return set(
      [cipd_tag.tag for cipd_tag in cipd_tags if cipd_tag.tag.startswith(tag)])


def fetch_and_upload_debug_symbols(api, project, import_in, remote, project_dir,
                                   packages, debug_symbol_attribute,
                                   debug_symbol_gcs_buckets):
  """
  Fetch debug symbol archives, unpack them, and upload debug symbols.

  Args:
    project (str): Jiri remote manifest project.
    import_in (str): Path to the edited manifest relative to $project
      containing debug symbol packages.
    remote (str): Remote manifest repository.
    project_dir (Path): Project root path of $import_in.
    packages (seq(str)): The list of CIPD packages updated in $import_in.
    debug_symbol_attribute (str): Jiri attribute to match debug symbol packages.
    debug_symbol_gcs_buckets (seq(str)): GCS buckets to upload debug symbols to.
  """
  with api.context(infra_steps=True):
    api.jiri.init(
        use_lock_file=True,
        attributes=(debug_symbol_attribute,),
    )
    # Fetch debug symbol packages using locally edited manifest.
    api.jiri.import_manifest(
        manifest=import_in,
        remote=remote,
        name=project,
    )
    api.jiri.fetch_packages(local_manifest=True)

  with api.step.nest('build'):
    gn_results = api.build.gen(
        checkout_root=api.path['start_dir'],
        fuchsia_build_dir=api.path['start_dir'].join('out', 'default'),
        target='x64',
        build_type='debug',
        product='products/bringup.gni',
        # //bundles:infratools is necessary to build upload_debug_symbols.
        packages=['//bundles:infratools'],
    )

    upload_debug_symbols_target = os.path.relpath(
        str(gn_results.tool('upload_debug_symbols')),
        str(gn_results.fuchsia_build_dir),
    )
    api.build.ninja(
        checkout_root=api.path['start_dir'],
        gn_results=gn_results,
        build_zircon=False,
        targets=[upload_debug_symbols_target],
    )

  build_id_dirs = []
  for package in packages:
    # Find archives for each debug symbol package.
    with api.context(cwd=project_dir):
      package_def = api.jiri.read_manifest_element(
          manifest=import_in,
          element_type='package',
          element_name=package,
      )
    # Skip non debug symbol packages.
    if debug_symbol_attribute not in package_def.get('attributes', ''):
      continue

    package_path = api.path['start_dir'].join(package_def['path'])
    archives = api.file.glob_paths(
        name='find archives for %s' % package,
        source=package_path,
        pattern='**/*.tar.bz2',
        test_data=(package_path.join('symbols.tar.bz2'),),
    )

    # Unpack archives into .build-id dirs.
    for archive in archives:
      # Extract API requires a unique, non-existent directory.
      archive_basename = os.path.basename(api.path.abspath(archive))
      output_dir = api.path['start_dir'].join(package, archive_basename)
      api.archive.extract(
          step_name='extract %s' % archive,
          archive_file=archive,
          output=output_dir,
      )
      build_id_dirs.append(output_dir)

  for debug_symbol_gcs_bucket in debug_symbol_gcs_buckets:
    api.upload_debug_symbols(
        step_name='upload debug symbols',
        upload_debug_symbols_path=gn_results.tool('upload_debug_symbols'),
        bucket=debug_symbol_gcs_bucket,
        build_id_dirs=build_id_dirs,
    )


def _get_platform_specific_packages(package, output):
  platform_regex = '(?<=' + package.replace('${platform}',
                                            r'\${platform=).*(?=})')
  pattern = re.compile(platform_regex)
  match = pattern.search(output)
  if match:
    platforms = match.group(0).split(',')
    return [package.replace('${platform}', platform) for platform in platforms]
  return [package]


def _append_urls(packages, old_version, new_version):
  package_line = '{package} old:{old} new:{new}'
  packages_with_urls = []
  for package in packages:
    if '${platform}' in package:
      packages_with_urls.append(package)
    else:
      packages_with_urls.append(
          package_line.format(
              old=CIPD_URL.format(package=package, version=old_version),
              new=CIPD_URL.format(package=package, version=new_version),
              package=package))
  return packages_with_urls


def RunSteps(api, project, manifest, remote, import_in, packages,
             debug_symbol_attribute, debug_symbol_gcs_buckets, lockfiles,
             dry_run, tag, ref, owners):
  with api.context(infra_steps=True):
    if owners:
      owners_step = api.step('owners', None)
      owners_step.presentation.step_summary_text = ', '.join(owners)

    api.jiri.init(use_lock_file=True)
    api.jiri.import_manifest(manifest, remote, project)
    api.jiri.update(run_hooks=False)
    api.jiri.run_hooks()

  project_dir = api.path['start_dir'].join(*project.split('/'))

  # Populate the set of tags.
  tags = get_tags_from_cipd(api, packages[0], tag, ref)

  # Get the intersection of tags.
  if (len(packages) > 1):
    for cipd_pkg in packages[1:]:
      tags = tags.intersection(get_tags_from_cipd(api, cipd_pkg, tag, ref))

  # Check if the intersection of tag exists.
  if not tags:
    api.step('unable to find common <tag> to roll', None)
    return

  # All the instances of tag point to the same 'instance_id'
  # Take the first one.
  version = list(tags)[0]

  with api.context(cwd=project_dir):
    changes = api.jiri.edit_manifest(
        import_in, packages=[(package, version) for package in packages])

    if not changes['packages']:
      api.step('manifest up-to-date; nothing to roll', None)
      return

    old_version = changes['packages'][0]['old_version']
    exact_packages = set()

    # Test data for jiri.resolve
    package_test_data = [
        ('@Subdir prebuilt/tools/buildbucket\n' +
         package[:package.index('${platform}')] +
         '${platform=linux-amd64,mac-amd64} git_revision:aa2dae..')
        for package in packages
        if '${platform}' in package
    ]
    test_data = '\n'.join(package_test_data)

    # Update the lockfiles.
    for lock_entry in lockfiles:
      fields = lock_entry.split('=')
      manifest = fields[0]
      lock = fields[1]
      resolve_output = api.jiri.resolve(
          local_manifest=True,
          output=lock,
          manifests=[manifest],
          step_test_data=test_data).stdout
      for p in packages:
        if '${platform}' in p:
          platform_pkgs = _get_platform_specific_packages(p, resolve_output)
          exact_packages = exact_packages.union(platform_pkgs)
        else:
          exact_packages.add(p)

    exact_packages = sorted(exact_packages)
    packages_with_urls = _append_urls(exact_packages, old_version, version)

    roller_name = api.buildbucket.builder_name.rstrip('-roller')
    message = COMMIT_MESSAGE.format(
        roller=roller_name,
        packages='\n'.join(packages_with_urls),
        old_version=old_version,
        version=version,
        builder=api.buildbucket.builder_name,
        build_id=api.buildbucket_util.id,
    )

    # Land the changes.
    rolled = api.auto_roller.attempt_roll(
        gerrit_project=project,
        repo_dir=project_dir,
        commit_message=message,
        dry_run=dry_run,
    )

  # If roll succeeded, upload any debug symbols that were rolled.
  # TODO(fxb/37432): Upload debug symbols with artifactory.
  if rolled and debug_symbol_gcs_buckets:
    with api.step.nest('fetch and upload debug symbols'):
      fetch_and_upload_debug_symbols(
          api=api,
          project=project,
          import_in=import_in,
          remote=remote,
          project_dir=project_dir,
          packages=packages,
          debug_symbol_attribute=debug_symbol_attribute,
          debug_symbol_gcs_buckets=debug_symbol_gcs_buckets,
      )


def GenTests(api):
  default_packages = [
      'chromium/fuchsia/webrunner-arm64', 'chromium/fuchsia/webrunner-amd64',
      'chromium/fuchsia/fidl'
  ]
  debug_symbol_packages = [
      'chromium/fuchsia/webrunner-debug-symbols-arm64',
      'chromium/fuchsia/webrunner-debug-symbols-amd64'
  ]
  platform_packages = [
      'fuchsia/tools/bb/${platform}', 'fuchsia/tools/luci/gsutil/${platform}'
  ]
  default_lockfiles = [
      'integration/flower=integration/jiri.lock',
  ]

  default_properties = api.properties(
      project='integration',
      manifest='minimal',
      remote='https://fuchsia.googlesource.com',
      import_in='chromium/chromium',
      packages=default_packages,
      lockfiles=default_lockfiles,
      owners=['nobody@google.com', 'noreply@google.com'])

  debug_symbols_properties = api.properties(
      project='integration',
      manifest='minimal',
      remote='https://fuchsia.googlesource.com',
      import_in='chromium/chromium',
      packages=default_packages + debug_symbol_packages,
      debug_symbol_gcs_buckets=('foo-bucket', 'bar-bucket'),
      lockfiles=default_lockfiles,
  )

  platform_pkg_properties = api.properties(
      project='integration',
      manifest='minimal',
      remote='https://fuchsia.googlesource.com',
      import_in='fuchsia/prebuilts',
      packages=platform_packages,
      lockfiles=default_lockfiles,
      tag='git_revision')

  yield (api.test('default') + default_properties + api.step_data(
      'cipd describe chromium/fuchsia/fidl',
      api.cipd.example_describe(
          package_name='chromium/fuchsia/fidl',
          version='GDGGW7Xs89z2apGaYf1mDvbQuHfYIoPexfedNzvKodUC',
          test_data_tags=[
              'version:70.0.3538.30',
              'version:70.0.3538.29',
          ])) + api.step_data(
              'cipd describe chromium/fuchsia/webrunner-amd64',
              api.cipd.example_describe(
                  package_name='chromium/fuchsia/webrunner-amd64',
                  version='r2S5xldLzzfJa2VzOYgoC6TsIWePSDLBI5FRywd_gHAC',
                  test_data_tags=[
                      'version:70.0.3538.30',
                  ])) +
         api.step_data(
             'cipd describe chromium/fuchsia/webrunner-arm64',
             api.cipd.example_describe(
                 package_name='chromium/fuchsia/webrunner-arm64',
                 version='mKI1nni0SW4F1cQAuYnkYU_RtDv47noSKO9vGHJVjzYC',
                 test_data_tags=[
                     'version:70.0.3538.30',
                 ])) + api.auto_roller.dry_run_step_data() +
         api.buildbucket.build(
             api.buildbucket.ci_build_message(builder='chromium-roller')))

  yield (api.test('no latest version match') + default_properties +
         api.step_data(
             'cipd describe chromium/fuchsia/fidl',
             api.cipd.example_describe(
                 package_name='chromium/fuchsia/fidl',
                 version='GDGGW7Xs89z2apGaYf1mDvbQuHfYIoPexfedNzvKodUC',
                 test_data_tags=[
                     'version:70.0.3538.30',
                 ])) + api.step_data(
                     'cipd describe chromium/fuchsia/webrunner-amd64',
                     api.cipd.example_describe(
                         package_name='chromium/fuchsia/webrunner-amd64',
                         version='r2S5xldLzzfJa2VzOYgoC6TsIWePSDLBI5FRywd_gHAC',
                         test_data_tags=[
                             'version:70.0.3538.30',
                         ])) +
         api.step_data(
             'cipd describe chromium/fuchsia/webrunner-arm64',
             api.cipd.example_describe(
                 package_name='chromium/fuchsia/webrunner-arm64',
                 version='mKI1nni0SW4F1cQAuYnkYU_RtDv47noSKO9vGHJVjzYC',
                 test_data_tags=[
                     'version:70.0.3538.29',
                 ])))

  yield (api.test('noop') + default_properties + api.step_data(
      'cipd describe chromium/fuchsia/fidl',
      api.cipd.example_describe(
          package_name='chromium/fuchsia/fidl',
          version='GDGGW7Xs89z2apGaYf1mDvbQuHfYIoPexfedNzvKodUC',
          test_data_tags=[
              'version:70.0.3538.30',
          ])) + api.step_data(
              'cipd describe chromium/fuchsia/webrunner-amd64',
              api.cipd.example_describe(
                  package_name='chromium/fuchsia/webrunner-amd64',
                  version='r2S5xldLzzfJa2VzOYgoC6TsIWePSDLBI5FRywd_gHAC',
                  test_data_tags=[
                      'version:70.0.3538.30',
                  ])) +
         api.step_data(
             'cipd describe chromium/fuchsia/webrunner-arm64',
             api.cipd.example_describe(
                 package_name='chromium/fuchsia/webrunner-arm64',
                 version='mKI1nni0SW4F1cQAuYnkYU_RtDv47noSKO9vGHJVjzYC',
                 test_data_tags=[
                     'version:70.0.3538.30',
                 ])) +
         api.step_data('jiri edit', api.json.output({'packages': []})))

  yield (api.test('default_with_platform') + platform_pkg_properties +
         api.step_data(
             'cipd describe fuchsia/tools/bb/${platform}',
             api.cipd.example_describe(
                 package_name='fuchsia/tools/bb/${platform}',
                 version='IX9BXIY7ZejVNabi6zF5lyq2KIW_ABFvBf1u3Cjr2dIC',
                 test_data_tags=[
                     'git_revision:aa2dae2ba04148bbe29726c64cfd9d40b6fc2616',
                 ])) +
         api.step_data(
             'cipd describe fuchsia/tools/luci/gsutil/${platform}',
             api.cipd.example_describe(
                 package_name='fuchsia/tools/luci/gsutil/${platform}',
                 version='IX9BXIY7ZejVNabi6zF5lyq2KIW_ABFvBf1u3Cjr2dIC',
                 test_data_tags=[
                     'git_revision:aa2dae2ba04148bbe29726c64cfd9d40b6fc2616',
                 ])) + api.auto_roller.dry_run_step_data() +
         api.buildbucket.build(
             api.buildbucket.ci_build_message(builder='tools-roller')))

  yield (api.test('default_platform_not_resolved') + platform_pkg_properties +
         api.step_data(
             'cipd describe fuchsia/tools/bb/${platform}',
             api.cipd.example_describe(
                 package_name='fuchsia/tools/bb/${platform}',
                 version='IX9BXIY7ZejVNabi6zF5lyq2KIW_ABFvBf1u3Cjr2dIC',
                 test_data_tags=[
                     'git_revision:aa2dae2ba04148bbe29726c64cfd9d40b6fc2616',
                 ])) +
         api.step_data(
             'cipd describe fuchsia/tools/luci/gsutil/${platform}',
             api.cipd.example_describe(
                 package_name='fuchsia/tools/luci/gsutil/${platform}',
                 version='IX9BXIY7ZejVNabi6zF5lyq2KIW_ABFvBf1u3Cjr2dIC',
                 test_data_tags=[
                     'git_revision:aa2dae2ba04148bbe29726c64cfd9d40b6fc2616',
                 ])) +
         api.step_data('jiri resolve', api.raw_io.stream_output('')) +
         api.auto_roller.dry_run_step_data() + api.buildbucket.build(
             api.buildbucket.ci_build_message(builder='tools-roller')))

  yield (api.test('with_debug_symbols') + debug_symbols_properties +
         api.step_data(
             'cipd describe chromium/fuchsia/fidl',
             api.cipd.example_describe(
                 package_name='chromium/fuchsia/fidl',
                 version='GDGGW7Xs89z2apGaYf1mDvbQuHfYIoPexfedNzvKodUC',
                 test_data_tags=[
                     'version:70.0.3538.30',
                     'version:70.0.3538.29',
                 ])) + api.step_data(
                     'cipd describe chromium/fuchsia/webrunner-amd64',
                     api.cipd.example_describe(
                         package_name='chromium/fuchsia/webrunner-amd64',
                         version='r2S5xldLzzfJa2VzOYgoC6TsIWePSDLBI5FRywd_gHAC',
                         test_data_tags=[
                             'version:70.0.3538.30',
                         ])) +
         api.step_data(
             'cipd describe chromium/fuchsia/webrunner-arm64',
             api.cipd.example_describe(
                 package_name='chromium/fuchsia/webrunner-arm64',
                 version='mKI1nni0SW4F1cQAuYnkYU_RtDv47noSKO9vGHJVjzYC',
                 test_data_tags=[
                     'version:70.0.3538.30',
                 ])) +
         api.step_data(
             'cipd describe chromium/fuchsia/webrunner-debug-symbols-amd64',
             api.cipd.example_describe(
                 package_name='chromium/fuchsia/webrunner-debug-symbols-amd64',
                 version='sI1KHrov4_lmsDNnkSJ0MHnKgEtWSl8XRCNRhpAhkacC',
                 test_data_tags=[
                     'version:70.0.3538.30',
                 ])) +
         api.step_data(
             'cipd describe chromium/fuchsia/webrunner-debug-symbols-arm64',
             api.cipd.example_describe(
                 package_name='chromium/fuchsia/webrunner-debug-symbols-arm64',
                 version='0soOR1_LGKIyPtlcA2mtD9dHQoKd0K1kkusVwcXLNLAC',
                 test_data_tags=[
                     'version:70.0.3538.30',
                 ])) + api.jiri.read_manifest_element(
                     api,
                     'chromium/chromium',
                     'package',
                     'chromium/fuchsia/webrunner-debug-symbols-amd64',
                     test_output={
                         'path': 'prebuilt/build_ids/amd64/webrunner',
                         'attributes': 'debug-symbols,debug-symbols-amd64',
                     },
                     nesting='fetch and upload debug symbols') +
         api.jiri.read_manifest_element(
             api,
             'chromium/chromium',
             'package',
             'chromium/fuchsia/webrunner-debug-symbols-arm64',
             test_output={
                 'path': 'prebuilt/build_ids/arm64/webrunner',
                 'attributes': 'debug-symbols,debug-symbols-amd64',
             },
             nesting='fetch and upload debug symbols') +
         api.jiri.read_manifest_element(
             api,
             'chromium/chromium',
             'package',
             'chromium/fuchsia/webrunner-arm64',
             test_output={
                 'path': 'chromium/fuchsia/arm64/webrunner',
             },
             nesting='fetch and upload debug symbols') +
         api.jiri.read_manifest_element(
             api,
             'chromium/chromium',
             'package',
             'chromium/fuchsia/webrunner-amd64',
             test_output={
                 'path': 'chromium/fuchsia/amd64/webrunner',
             },
             nesting='fetch and upload debug symbols') +
         api.jiri.read_manifest_element(
             api,
             'chromium/chromium',
             'package',
             'chromium/fuchsia/fidl',
             test_output={
                 'path': 'chromium/fuchsia/fidl',
             },
             nesting='fetch and upload debug symbols') +
         api.auto_roller.success_step_data() + api.buildbucket.build(
             api.buildbucket.ci_build_message(builder='chromium-roller')))