# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building goma toolchain images."""

from recipe_engine.recipe_api import Property

DEPS = [
    'fuchsia/docker',
    'fuchsia/gcloud',
    'fuchsia/git',
    'fuchsia/goma',
    'fuchsia/gsutil',
    'recipe_engine/archive',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/raw_io',
    'recipe_engine/step',
    'recipe_engine/time',
    'recipe_engine/url',
]

PROPERTIES = {
    'package':
        Property(kind=str, help='cipd path of the toolchain package'),
    'version':
        Property(
            kind=str,
            help='cipd version of the toolchain package',
            default='git_revision:1475fad1d0d45572471575cdda8fed6eeab5b0ec'),
    'toolchain':
        Property(kind=str, help='type of the toolchain', default='clang'),
    'platform':
        Property(kind=str, help='toolchain host platform', default=None),
    'domain':
        Property(
            kind=str,
            help='domain name of container registry',
            default='gcr.io'),
    'project':
        Property(
            kind=str, help='name of goma cloud project',
            default='goma-fuchsia'),
    'root':
        Property(
            kind=str,
            help='root directory name for container registry',
            default='fuchsia_linux'),
    'update_goma_base_images':
        Property(
            kind=bool,
            help='Update goma base images even if they already exist',
            default=False),
}

GOMATOOLS_VERSION = 'git_revision:22684ee9174606073a2604e1d58fbf03ba3ad564'


def ensure_gomatools(api, gomatools_dir):
  with api.step.nest('ensure gomatools'):
    pkgs = api.cipd.EnsureFile()
    pkgs.add_package(
        'fuchsia_internal/third_party/goma/server/gomatools/linux-amd64',
        GOMATOOLS_VERSION)
    api.cipd.ensure(gomatools_dir, pkgs)


def generate_time_stamp(api):
  return "{:%Y%m%d_%H%M%S}".format(api.time.utcnow())


def build_goma_base_image(api, dockerfile, image, timestamp=None):
  api.file.copy('copy dockerfile', api.resource(dockerfile),
                api.path['cleanup'].join(dockerfile))
  tags = [image]
  if timestamp:
    tags.append(image + ':' + timestamp)
  with api.context(cwd=api.path['cleanup']):
    api.docker.build(
        dockerfile=api.path['cleanup'].join(dockerfile),
        tags=tags,
    )
  api.docker('push', image)


def ensure_goma_base_images(api,
                            domain,
                            project_id,
                            root,
                            timestamp=None,
                            update_if_exist=False):
  """Ensure the base goma images in container registry.

  E.g. ensure_goma_base_images(api,
                               'gcr.io',\
                               'goma-fuchsia',\
                               'fuchsia_linux')
      will create docker images at
      gcr.io/goma-fuchsia/fuchsia_linux/base,
      gcr.io/goma-fuchsia/fuchsia_linux/runtime,
      gcr.io/goma-fuchsia/fuchsia_linux/remoteexec-platform,
      using 'debian:buster-slim' as bottom layer image.

  Args:
      * domain (str) - The domain name of the container register.
      * project (str) - The gcloud project ID.
      * root (str) - The name of container registry root.
      * update_if_exist (str) - Update the container if it exists.
  """
  with api.step.nest('ensure goma base images'):
    # Build goma base image.
    goma_base_image = api.url.join(domain, project_id, root, 'base')
    if not api.gcloud.container_image_exists(
        goma_base_image) or update_if_exist:
      build_goma_base_image(api, 'Dockerfile_goma_base', goma_base_image,
                            timestamp)
    # Build goma runtime image.
    runtime_image = api.url.join(domain, project_id, root, 'runtime')
    if not api.gcloud.container_image_exists(runtime_image) or update_if_exist:
      api.docker('pull', goma_base_image)
      api.docker('tag', goma_base_image, 'base')
      build_goma_base_image(api, 'Dockerfile_goma_runtime', runtime_image,
                            timestamp)
    # Build goma remoteexec-platform
    remote_image = api.url.join(domain, project_id, root, 'remoteexec-platform')
    if not api.gcloud.container_image_exists(remote_image) or update_if_exist:
      build_goma_base_image(api, 'Dockerfile_goma_remoteexec_platform',
                            remote_image, timestamp)


def push_toolchain_to_goma(api,
                           goma_temp_dir,
                           domain,
                           project_id,
                           root,
                           toolchain,
                           toolchain_version,
                           container_version='latest',
                           stamp=None):
  """Push and register the toolchain docker image with Goma.

  E.g. push_toolchain_to_goma('gcr.io',\
                              goma_temp_dir,\
                              'goma-fuchsia',\
                              'fuchsia_linux',\
                              'clang',\
                              'some_hash',\
                              'latest',
                              '20190909_122500')
      will copy the toolchain binaries from docker image
      'gcr.io/goma-fuchsia/fuchsia_linux/clang-some_hash:latest'
      to appropriate gs bucket of the Goma gcloud project and
      register it with Goma server.

  Args:
    * domain (str) - The domain name of the clontainer register.
    * project_id (str) - The gcloud project ID.
    * root (str) - The root path of all toolchains. E.g. 'fuchsia_linux'.
    * toolchain (str) - The name of the toolchain.
    * toolchain_version (str) - The version string of the toolchain.
                                Usually it is the commit hash.
    * container_version (str) - The container image version. 'latest' by default.
    * stamp (str) - Time stamp string.
  """
  with api.step.nest('push toolchain to goma'):
    # URI = ${domain}/${project_id}/${root}/${toolchain}-${toolchain_version}:${container_version}
    # e.g. URI = gcr.io/goma-fuchsia/fuchsia_linux/clang-0a5975a5bcd59cb07efdc86b47498fbb7c095ab:latest
    runtime_image = api.url.join(domain, project_id, root, 'runtime')
    if stamp is None:
      stamp = generate_time_stamp(api)  # pragma: no cover
    # extract-goma-files
    toolchain_tag = '%s-%s' % (toolchain, toolchain_version)
    toolchain_image = api.url.join(domain, project_id, root, toolchain_tag)
    with api.docker.create(toolchain_image + ':' +
                           container_version) as temp_copy:
      api.docker.copy(temp_copy, ['/var/cache/goma', '/etc/goma/descriptors'],
                      goma_temp_dir)
    api.file.move('rename cmdstorage', goma_temp_dir.join('goma', 'cmdstorage'),
                  goma_temp_dir.join('goma', 'sha256'))

    # pull-toolchain-runtime
    api.docker('pull', runtime_image)
    api.docker('tag', runtime_image, 'toolchain/runtime')

    api.file.copy('copy dockerfile',
                  api.resource('Dockerfile_fuchsia_toolchain_push_test'),
                  goma_temp_dir)
    # build-test-image
    local_toolchain_tag = api.url.join(root, toolchain_tag)
    # Set env
    with api.context(cwd=goma_temp_dir):
      api.docker.build(
          dockerfile=goma_temp_dir.join(
              "Dockerfile_fuchsia_toolchain_push_test"),
          tags=['%s_test' % toolchain_image,
                '%s_test' % local_toolchain_tag],
      )
    # run test
    api.docker('run', '%s_test' % local_toolchain_tag)

    # store-files
    api.gsutil.upload(
        bucket='%s-files' % project_id,
        src=goma_temp_dir.join('goma', 'sha256'),
        dst='',
        recursive=True,
        multithreaded=True,
    )

    api.gsutil.upload(
        bucket='%s-toolchain-config' % project_id,
        src=goma_temp_dir.join('descriptors'),
        dst='%s/%s/' % (root, toolchain_tag),
        recursive=True,
        multithreaded=True,
    )

    api.file.remove('remove seq file', goma_temp_dir.join('seq'))
    api.file.write_text('write seq file', goma_temp_dir.join('seq'), stamp)

    api.gsutil.upload(
        bucket='%s-toolchain-config' % project_id,
        src=goma_temp_dir.join('seq'),
        dst='%s/seq' % root,
        recursive=False,
        multithreaded=True,
    )


def RunSteps(api, package, version, toolchain, platform, domain, project, root,
             update_goma_base_images):
  assert package.endswith('${platform}')
  # TODO: factor this out into a host_build recipe module.
  host_platform = '%s-%s' % (api.platform.name.replace('win', 'windows'), {
      'intel': {
          32: '386',
          64: 'amd64',
      },
      'arm': {
          32: 'armv6',
          64: 'arm64',
      },
  }[api.platform.arch][api.platform.bits])
  target_platform = platform or host_platform
  pkgs = api.cipd.EnsureFile()
  platforms = [target_platform]
  if target_platform == 'mac-amd64':
    # Cross compilation is required, packing
    # linux toolchain as well.
    platforms.append('linux-amd64')

  for item in platforms:
    pkgs.add_package(package.replace('${platform}', item), version, item)
  toolchain_dir = api.path['cleanup'].join('toolchain')
  with api.step.nest('ensure toolchain'):
    api.cipd.ensure(toolchain_dir, pkgs)

  timestamp = generate_time_stamp(api)
  if version.startswith('git_revision:'):
    version = version[len('git_revision:'):]

  goma_temp_dir = api.path['cleanup'].join('goma')
  ensure_gomatools(api, goma_temp_dir)
  with api.step.nest('preprocess toolchain'):
    if toolchain == 'clang':
      dest_dir = api.path['cleanup'].join('toolchain_copy')
      py_args = [
          str(toolchain_dir),
          str(dest_dir),
      ]
      for item in platforms:
        py_args.append('--platform=%s' % item)
      api.python(
          'minimize clang',
          script=api.resource('preprocess_clang.py'),
          args=py_args,
      )
      toolchain_dir = dest_dir
    for item in platforms:
      api.archive.package(toolchain_dir.join(item)).archive(
          'pack toolchain package',
          goma_temp_dir.join('{}-{}.tgz'.format(toolchain, item)))

  toolchain_tag = '%s-%s-%s' % (toolchain, version, target_platform)
  container_tag = api.url.join(domain, project, root, toolchain_tag)

  with api.step.nest('build toolchain image'):
    try:
      ensure_goma_base_images(api, domain, project, root, timestamp,
                              update_goma_base_images)
      api.docker(
          'pull',
          api.url.join(domain, project, root, 'base'),
          step_name='pull base')

      api.docker(
          'tag',
          api.url.join(domain, project, root, 'base'),
          'base',
          step_name='tag base')

      with api.step.nest('copy goma docker configurations'):
        dockerfile_platform = target_platform.replace('-', '_')
        dockerfile = 'Dockerfile_fuchsia_toolchain_{}'.format(
            dockerfile_platform)
        api.file.copy('copy dockerfile', api.resource(dockerfile),
                      goma_temp_dir.join(dockerfile))
        setupfile = 'setup_fuchsia_{}_{}'.format(toolchain, dockerfile_platform)
        api.file.copy('copy toolchain setup file', api.resource(setupfile),
                      goma_temp_dir.join('setup'))
      with api.context(cwd=goma_temp_dir):
        api.docker.build(
            dockerfile=dockerfile,
            tags=[container_tag, container_tag + ':' + timestamp],
            build_args=['TOOLCHAIN=%s' % toolchain])
      api.docker('push', container_tag)
      api.docker('push', container_tag + ':' + timestamp)
      push_toolchain_to_goma(
          api,
          goma_temp_dir,
          domain,
          project,
          root,
          toolchain,
          version + '-' + target_platform,
          stamp=timestamp)
    finally:
      api.docker.cleanup()


def GenTests(api):
  default_properties = api.properties(
      package='fuchsia/third_party/clang/${platform}',
      version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e',
      toolchain='clang',
      domain='gcr.io',
      project='goma-fuchsia',
      root='fuchsia_linux')

  properties_missing_platform = api.properties(
      package='fuchsia/third_party/clang/linux-amd64',
      version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e',
      toolchain='clang',
      domain='gcr.io',
      project='goma-fuchsia',
      root='fuchsia_linux')

  wrong_platform_properties = api.properties(
      package='fuchsia/third_party/clang/${platform}',
      version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e',
      toolchain='clang',
      platform='linux_amd64',
      domain='gcr.io',
      project='goma-fuchsia',
      root='fuchsia_linux')

  cross_compile_properties = api.properties(
      package='fuchsia/third_party/clang/${platform}',
      version='git_revision:56e9b608ad3a42d59cd4521b0eb495388261c67e',
      toolchain='clang',
      platform='mac-amd64',
      domain='gcr.io',
      project='goma-fuchsia',
      root='fuchsia_linux')

  yield api.test('default') + default_properties + api.buildbucket.try_build(
      git_repo='https://fuchsia.googlesource.com/integration')

  yield api.test(
      'base image missing'
  ) + default_properties + api.buildbucket.try_build(
      git_repo='https://fuchsia.googlesource.com/integration'
  ) + api.step_data(
      'build toolchain image.ensure goma base images.check container existence',
      retcode=1
  ) + api.step_data(
      'build toolchain image.ensure goma base images.check container existence (2)',
      retcode=1
  ) + api.step_data(
      'build toolchain image.ensure goma base images.check container existence (3)',
      retcode=1)

  yield api.test('missing_platform'
                ) + properties_missing_platform + api.buildbucket.try_build(
                    git_repo='https://fuchsia.googlesource.com/integration'
                ) + api.expect_exception('AssertionError')

  yield api.test(
      'wrong_platform') + wrong_platform_properties + api.buildbucket.try_build(
          git_repo='https://fuchsia.googlesource.com/integration')

  yield api.test(
      'cross_compile') + cross_compile_properties + api.buildbucket.try_build(
          git_repo='https://fuchsia.googlesource.com/integration')