# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building goma toolchain images."""

from recipe_engine.config import List
from recipe_engine.recipe_api import Property

DEPS = [
    'fuchsia/docker',
    'fuchsia/gcloud',
    'fuchsia/git',
    'fuchsia/goma',
    'fuchsia/gsutil',
    'recipe_engine/archive',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/raw_io',
    'recipe_engine/step',
    'recipe_engine/time',
    'recipe_engine/url',
]

PROPERTIES = {
    'package':
        Property(
            kind=str,
            help='cipd path of the toolchain package',
            default='fuchsia/third_party/clang'),
    'version':
        Property(
            kind=str,
            help='cipd version of the toolchain package',
            default='git_revision:1475fad1d0d45572471575cdda8fed6eeab5b0ec'),
    'toolchain':
        Property(kind=str, help='type of the toolchain', default='clang'),
    'platforms':
        Property(
            kind=List(str),
            help='toolchain host platform',
            default=['linux-amd64', 'mac-amd64']),
    'domain':
        Property(
            kind=str,
            help='domain name of container registry',
            default='gcr.io'),
    'project':
        Property(
            kind=str, help='name of goma cloud project',
            default='goma-fuchsia'),
    'root':
        Property(
            kind=str,
            help='root directory name for container registry',
            default='fuchsia_linux'),
    'update_goma_base_images':
        Property(
            kind=bool,
            help='Update goma base images even if they already exist',
            default=False),
}

GOMATOOLS_VERSION = 'git_revision:22684ee9174606073a2604e1d58fbf03ba3ad564'


def ensure_gomatools(api, gomatools_dir):
  with api.step.nest('ensure gomatools'):
    pkgs = api.cipd.EnsureFile()
    pkgs.add_package(
        'fuchsia_internal/third_party/goma/server/gomatools/linux-amd64',
        GOMATOOLS_VERSION)
    api.cipd.ensure(gomatools_dir, pkgs)


def generate_time_stamp(api):
  return "{:%Y%m%d_%H%M%S}".format(api.time.utcnow())


def build_goma_base_image(api, dockerfile, image, timestamp=None):
  api.file.copy('copy dockerfile', api.resource(dockerfile),
                api.path['cleanup'].join(dockerfile))
  tags = [image]
  if timestamp:
    tags.append(image + ':' + timestamp)
  with api.context(cwd=api.path['cleanup']):
    api.docker.build(
        dockerfile=api.path['cleanup'].join(dockerfile),
        tags=tags,
    )
  api.docker('push', image)


def ensure_goma_base_images(api,
                            domain,
                            project_id,
                            root,
                            timestamp=None,
                            update_if_exist=False):
  """Ensure the base goma images in container registry.

  E.g. ensure_goma_base_images(api,
                               'gcr.io',\
                               'goma-fuchsia',\
                               'fuchsia_linux')
      will create docker images at
      gcr.io/goma-fuchsia/fuchsia_linux/base,
      gcr.io/goma-fuchsia/fuchsia_linux/runtime,
      gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform,
      using 'debian:buster-slim' as bottom layer image.

  Args:
      * domain (str) - The domain name of the container register.
      * project (str) - The gcloud project ID.
      * root (str) - The name of container registry root.
      * update_if_exist (str) - Update the container if it exists.
  """
  with api.step.nest('ensure goma base images'):
    # Build goma base image.
    goma_base_image = api.url.join(domain, project_id, root, 'base')
    if not api.gcloud.container_image_exists(
        goma_base_image) or update_if_exist:
      build_goma_base_image(api, 'Dockerfile_goma_base', goma_base_image,
                            timestamp)
    # Build goma runtime image.
    runtime_image = api.url.join(domain, project_id, root, 'runtime')
    if not api.gcloud.container_image_exists(runtime_image) or update_if_exist:
      api.docker('pull', goma_base_image)
      api.docker('tag', goma_base_image, 'base')
      build_goma_base_image(api, 'Dockerfile_goma_runtime', runtime_image,
                            timestamp)
    # pull-toolchain-runtime
    api.docker('pull', runtime_image)
    api.docker('tag', runtime_image, 'toolchain/runtime')
    # Build goma remoteexec-platform
    remote_image = api.url.join(domain, project_id, root, 'remoteexec-platform')
    if not api.gcloud.container_image_exists(remote_image) or update_if_exist:
      build_goma_base_image(api, 'Dockerfile_goma_remoteexec_platform',
                            remote_image, timestamp)


def push_toolchain_to_goma(api,
                           goma_temp_dir,
                           project_id,
                           root,
                           toolchain_tag,
                           container_tag,
                           container_version='latest',
                           stamp=None):
  """Push and register the toolchain docker image with Goma.

  E.g. push_toolchain_to_goma(goma_temp_dir,\
                              'goma-fuchsia',\
                              'fuchsia_linux',\
                              'clang-some_hash-linux-amd64',\
                              'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash-linux-amd64',\
                              'latest',
                              '20190909_122500')
      will copy the toolchain binaries from docker image
      'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash-linux-amd64:latest'
      to appropriate gs bucket of the Goma gcloud project and
      register it with Goma server.

  Args:
    * goma_temp_dir (str) - The temp directory for Goma resources.
    * project_id (str) - The gcloud project ID.
    * root (str) - The root path of all toolchains. E.g. 'fuchsia_linux'.
    * toolchain_tag (str) - Toolchain version and platform tag.
    * container_tag (str) - Toolchain container tag.
    * container_version (str) - The container image version. 'latest' by default.
    * stamp (str) - Time stamp string.
  """
  with api.step.nest('push toolchain to goma'):
    # URI = ${domain}/${project_id}/${root}/${toolchain}-${toolchain_version}:${container_version}
    # e.g. URI = gcr.io/goma-fuchsia/fuchsia_linux/clang-0a5975a5bcd59cb07efdc86b47498fbb7c095ab:latest
    if stamp is None:
      stamp = generate_time_stamp(api)  # pragma: no cover
    # extract-goma-files
    with api.docker.create(container_tag + ':' +
                           container_version) as temp_copy:
      api.docker.copy(temp_copy, ['/var/cache/goma', '/etc/goma/descriptors'],
                      goma_temp_dir)
    api.file.move('rename cmdstorage', goma_temp_dir.join('goma', 'cmdstorage'),
                  goma_temp_dir.join('goma', 'sha256'))
    api.file.remove(
        'remove dockerfile if it exists',
        goma_temp_dir.join('Dockerfile_fuchsia_toolchain_push_test'))
    api.file.copy('copy dockerfile',
                  api.resource('Dockerfile_fuchsia_toolchain_push_test'),
                  goma_temp_dir)
    # build-test-image
    local_toolchain_tag = api.url.join(root, toolchain_tag)
    # Set env
    with api.context(cwd=goma_temp_dir):
      api.docker.build(
          dockerfile=goma_temp_dir.join(
              "Dockerfile_fuchsia_toolchain_push_test"),
          tags=['%s_test' % container_tag,
                '%s_test' % local_toolchain_tag],
      )
    # run test
    api.docker('run', '%s_test' % local_toolchain_tag)

    # store-files
    api.gsutil.upload(
        bucket='%s-files' % project_id,
        src=goma_temp_dir.join('goma', 'sha256'),
        dst='',
        recursive=True,
        multithreaded=True,
    )

    api.gsutil.upload(
        bucket='%s-toolchain-config' % project_id,
        src=goma_temp_dir.join('descriptors'),
        dst='%s/%s/' % (root, toolchain_tag),
        recursive=True,
        multithreaded=True,
    )

    api.file.remove('remove seq file', goma_temp_dir.join('seq'))
    api.file.write_text('write seq file', goma_temp_dir.join('seq'), stamp)

    api.gsutil.upload(
        bucket='%s-toolchain-config' % project_id,
        src=goma_temp_dir.join('seq'),
        dst='%s/seq' % root,
        recursive=False,
        multithreaded=True,
    )


def build_and_push_goma_toolchain(api, toolchain, version, domain, project,
                                  root, platform, goma_temp_dir, timestamp):
  toolchain_tag = '%s-%s-%s' % (toolchain, version, platform)
  container_tag = api.url.join(domain, project, root, toolchain_tag)
  with api.step.nest('copy goma docker configurations'):
    dockerfile_platform = platform.replace('-', '_')
    dockerfile = 'Dockerfile_fuchsia_toolchain_{}'.format(dockerfile_platform)
    api.file.copy('copy dockerfile', api.resource(dockerfile),
                  goma_temp_dir.join(dockerfile))
    setupfile = 'setup_fuchsia_{}_{}'.format(toolchain, dockerfile_platform)
    # On CI bots, file.copy complains error 13 if the setup file already
    # exists. Remove it first to avoid this issue.
    api.file.remove('remove existing setup file', goma_temp_dir.join('setup'))
    api.file.copy('copy toolchain setup file', api.resource(setupfile),
                  goma_temp_dir.join('setup'))
    with api.context(cwd=goma_temp_dir):
      api.docker.build(
          dockerfile=dockerfile,
          tags=[container_tag, container_tag + ':' + timestamp],
          build_args=['TOOLCHAIN=%s' % toolchain])
      api.docker('push', container_tag)
      api.docker('push', container_tag + ':' + timestamp)
      push_toolchain_to_goma(
          api,
          goma_temp_dir,
          project,
          root,
          toolchain_tag,
          container_tag,
          stamp=timestamp)


def RunSteps(api, package, version, toolchain, platforms, domain, project, root,
             update_goma_base_images):
  preprocess_platforms = platforms[:]
  if 'mac-amd64' in preprocess_platforms and 'linux-amd64' not in preprocess_platforms:
    # To cross compile mac target on goma linux workers, both mac and linux
    # toolchains are required.
    preprocess_platforms.append('linux-amd64')
  pkgs = api.cipd.EnsureFile()
  for platform in preprocess_platforms:
    pkgs.add_package(api.url.join(package, platform), version, platform)
  toolchain_dir = api.path['cleanup'].join('toolchain')
  with api.step.nest('ensure toolchain'):
    api.cipd.ensure(toolchain_dir, pkgs)

  timestamp = generate_time_stamp(api)
  if version.startswith('git_revision:'):
    version = version[len('git_revision:'):]

  goma_temp_dir = api.path['cleanup'].join('goma')
  ensure_gomatools(api, goma_temp_dir)
  with api.step.nest('preprocess toolchain'):
    if toolchain == 'clang':
      dest_dir = api.path['cleanup'].join('toolchain_copy')
      api.python(
          'preprocess clang',
          script=api.resource('preprocess_clang.py'),
          args=[
              toolchain_dir,
              dest_dir,
          ] + ['--platform=%s' % platform for platform in preprocess_platforms],
      )
      toolchain_dir = dest_dir
    for platform in preprocess_platforms:
      api.archive.package(toolchain_dir.join(platform)).archive(
          'pack toolchain package for %s' % platform,
          goma_temp_dir.join('{}-{}.tgz'.format(toolchain, platform)))

  ensure_goma_base_images(api, domain, project, root, timestamp,
                          update_goma_base_images)
  api.docker(
      'pull',
      api.url.join(domain, project, root, 'base'),
      step_name='pull base')
  api.docker(
      'tag',
      api.url.join(domain, project, root, 'base'),
      'base',
      step_name='tag base')
  try:
    for platform in platforms:
      with api.step.nest('build toolchain image for %s' % platform):
        build_and_push_goma_toolchain(api, toolchain, version, domain, project,
                                      root, platform, goma_temp_dir, timestamp)
  finally:
    api.docker.cleanup()


def GenTests(api):
  default_properties = api.properties(
      package='fuchsia/third_party/clang',
      version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e',
      toolchain='clang',
      domain='gcr.io',
      project='goma-fuchsia',
      root='fuchsia_linux')

  properties_missing_platform = api.properties(
      package='fuchsia/third_party/clang/linux-amd64',
      version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e',
      toolchain='clang',
      domain='gcr.io',
      project='goma-fuchsia',
      root='fuchsia_linux')

  wrong_platform_properties = api.properties(
      package='fuchsia/third_party/clang',
      version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e',
      toolchain='clang',
      platforms=['linux_amd64'],
      domain='gcr.io',
      project='goma-fuchsia',
      root='fuchsia_linux')

  cross_compile_properties = api.properties(
      package='fuchsia/third_party/clang',
      version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e',
      toolchain='clang',
      platforms=['mac-amd64'],
      domain='gcr.io',
      project='goma-fuchsia',
      root='fuchsia_linux')

  yield api.test('default') + default_properties + api.buildbucket.try_build(
      git_repo='https://fuchsia.googlesource.com/integration')

  yield api.test(
      'base image missing'
  ) + default_properties + api.buildbucket.try_build(
      git_repo='https://fuchsia.googlesource.com/integration'
  ) + api.step_data(
      'ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/runtime',
      retcode=1
  ) + api.step_data(
      'ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/base',
      retcode=1
  ) + api.step_data(
      'ensure goma base images.check existence of gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform',
      retcode=1)

  yield api.test(
      'wrong_platform') + wrong_platform_properties + api.buildbucket.try_build(
          git_repo='https://fuchsia.googlesource.com/integration')

  yield api.test(
      'cross_compile') + cross_compile_properties + api.buildbucket.try_build(
          git_repo='https://fuchsia.googlesource.com/integration')