# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building Clang toolchain."""

from recipe_engine.recipe_api import Property
from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest

from google.protobuf import json_format

import re

DEPS = [
    'fuchsia/git',
    'fuchsia/gitiles',
    'fuchsia/goma',
    'fuchsia/gsutil',
    'fuchsia/macos_sdk',
    'fuchsia/upload',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/isolated',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/runtime',
    'recipe_engine/raw_io',
    'recipe_engine/scheduler',
    'recipe_engine/source_manifest',
    'recipe_engine/step',
]

TARGET_TO_ARCH = {
    'x64': 'x86_64',
    'arm64': 'aarch64',
}
TARGETS = TARGET_TO_ARCH.keys()

PLATFORM_TO_TRIPLE = {
    'linux-amd64': 'x86_64-linux-gnu',
    'linux-arm64': 'aarch64-linux-gnu',
    'mac-amd64': 'x86_64-apple-darwin',
}
PLATFORMS = PLATFORM_TO_TRIPLE.keys()

# TODO(TC-476): remove this and switch to normalized triples.
TRIPLE_TO_NORMALIZED_TRIPLE = {
    'x86_64-linux-gnu': 'x86_64-unknown-linux-gnu',
    'aarch64-linux-gnu': 'aarch64-unknown-linux-gnu',
    'x86_64-apple-darwin': 'x86_64-apple-darwin',
}

LIBXML2_GIT = 'https://fuchsia.googlesource.com/third_party/libxml2'
ZLIB_GIT = 'https://fuchsia.googlesource.com/third_party/zlib'

CIPD_SERVER_HOST = 'chrome-infra-packages.appspot.com'

PROPERTIES = {
    'repository':
        Property(
            kind=str,
            help='Git repository URL',
            default='https://fuchsia.googlesource.com/third_party/llvm-project'
        ),
    'revision':
        Property(kind=str, help='Git revision', default=None),
    'platform':
        Property(kind=str, help='CIPD platform for the target', default=None),
}


def build_zlib(api, destdir, manifest, env=None):
  zlib_dir = api.path['start_dir'].join('zlib')
  src_dir = zlib_dir.join('src')
  revision = api.git.checkout(ZLIB_GIT, src_dir, ref='refs/tags/v1.2.9')
  git_checkout = manifest.directories[str(src_dir)].git_checkout
  git_checkout.repo_url = ZLIB_GIT
  git_checkout.revision = revision
  git_checkout.fetch_ref = 'refs/tags/v1.2.9'
  obj_dir = zlib_dir.join('obj')
  api.file.ensure_directory('make objdir', obj_dir)
  with api.context(cwd=obj_dir, env=env):
    api.step('configure', [
        src_dir.join('configure'),
        '--prefix=',
        '--static',
    ])
    api.step('build', ['make', '-j%d' % api.goma.jobs])
    api.step('install', ['make', 'install', 'DESTDIR=%s' % destdir])


def build_libxml2(api, host, target, destdir, manifest, env=None):
  libxml2_dir = api.path['start_dir'].join('libxml2')
  src_dir = libxml2_dir.join('src')
  revision = api.git.checkout(LIBXML2_GIT, src_dir, ref='refs/tags/v2.9.8')
  git_checkout = manifest.directories[str(src_dir)].git_checkout
  git_checkout.repo_url = LIBXML2_GIT
  git_checkout.revision = revision
  git_checkout.fetch_ref = 'refs/tags/v2.9.8'
  with api.context(cwd=src_dir):
    api.step('autoconf', ['autoreconf', '-i', '-f'])
  obj_dir = libxml2_dir.join('obj')
  api.file.ensure_directory('make objdir', obj_dir)
  with api.context(cwd=obj_dir):
    api.step('configure', [
        src_dir.join('configure'),
        '--prefix=',
        '--build=%s' % host,
        '--host=%s' % target,
        '--disable-shared',
        '--enable-static',
        '--with-zlib=%s' % destdir,
        '--without-icu',
        '--without-lzma',
        '--without-python',
    ] + ['%s=%s' % (k, v) for k, v in env.iteritems()])
    api.step('build', ['make', '-j%d' % api.goma.jobs])
    api.step('install', ['make', 'install', 'DESTDIR=%s' % destdir])


def RunSteps(api, repository, revision, platform):
  api.goma.ensure()

  prod = api.buildbucket.builder_id.bucket == 'prod'
  ci = api.buildbucket.builder_id.bucket == 'ci'

  gitiles_commit = api.buildbucket.build_input.gitiles_commit
  if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id:
    repository = 'https://%s/%s' % (gitiles_commit.host, gitiles_commit.project)
    revision = gitiles_commit.id

  # TODO: factor this out into a host_build recipe module.
  host_platform = '%s-%s' % (api.platform.name.replace('win', 'windows'), {
      'intel': {
          32: '386',
          64: 'amd64',
      },
      'arm': {
          32: 'armv6',
          64: 'arm64',
      },
  }[api.platform.arch][api.platform.bits])
  target_platform = platform or host_platform

  manifest = Manifest()

  with api.step.nest('ensure_packages'):
    with api.context(infra_steps=True):
      cipd_dir = api.path['start_dir'].join('cipd')
      pkgs = api.cipd.EnsureFile()
      pkgs.add_package('infra/cmake/${platform}', 'version:3.9.2')
      pkgs.add_package('infra/ninja/${platform}', 'version:1.8.2')
      pkgs.add_package('fuchsia/third_party/clang/${platform}', 'goma')
      pkgs.add_package('fuchsia/sysroot/linux-386', 'latest', 'linux-i386')
      pkgs.add_package('fuchsia/sysroot/linux-amd64', 'latest', 'linux-amd64')
      pkgs.add_package('fuchsia/sysroot/linux-arm64', 'latest', 'linux-arm64')
      pkgs.add_package('fuchsia/sysroot/linux-armv6l', 'latest', 'linux-armhf')
      pkgs.add_package('fuchsia/sdk/core/${platform}', 'latest', 'sdk')
      ensured = api.cipd.ensure(cipd_dir, pkgs)
      for subdir, pins in ensured.iteritems():
        directory = manifest.directories[str(cipd_dir.join(subdir))]
        directory.cipd_server_host = CIPD_SERVER_HOST
        for pin in pins:
          directory.cipd_package[pin.package].instance_id = pin.instance_id

  staging_dir = api.path.mkdtemp('clang')
  pkg_name = 'clang-%s' % api.platform.name.replace('mac', 'darwin')
  pkg_dir = staging_dir.join(pkg_name)
  api.file.ensure_directory('create pkg dir', pkg_dir)

  with api.context(infra_steps=True):
    llvm_dir = api.path['start_dir'].join('llvm-project')
    revision = api.git.checkout(
        repository,
        path=llvm_dir,
        ref=revision,
        step_test_data=lambda: api.raw_io.test_api.stream_output(revision))
    git_checkout = manifest.directories[str(llvm_dir)].git_checkout
    git_checkout.repo_url = repository
    git_checkout.revision = revision

  lib_install_dir = staging_dir.join('lib_install')
  api.file.ensure_directory('create lib_install_dir', lib_install_dir)

  target_triple = PLATFORM_TO_TRIPLE[target_platform]
  host_triple = PLATFORM_TO_TRIPLE[host_platform]

  with api.macos_sdk(), api.goma.build_with_goma():
    if api.platform.name == 'linux':
      host_sysroot = cipd_dir.join(host_platform)
      target_sysroot = cipd_dir.join(target_platform)
    elif api.platform.name == 'mac':
      # TODO(IN-148): Eventually use our own hermetic sysroot as for Linux.
      step_result = api.step(
          'xcrun', ['xcrun', '--show-sdk-path'],
          stdout=api.raw_io.output(name='sdk-path', add_output_log=True),
          step_test_data=lambda: api.raw_io.test_api.stream_output(
              '/some/xcode/path'))
      target_sysroot = host_sysroot = step_result.stdout.strip()
    else:  # pragma: no cover
      assert False, 'unsupported platform'

    env = {
        'CC':
            '%s %s' %
            (api.goma.goma_dir.join('gomacc'), cipd_dir.join('bin', 'clang')),
        'CFLAGS':
            '-O3 -fPIC --target=%s --sysroot=%s' %
            (target_triple, target_sysroot),
        'AR':
            cipd_dir.join('bin', 'llvm-ar'),
        'NM':
            cipd_dir.join('bin', 'llvm-nm'),
        'RANLIB':
            cipd_dir.join('bin', 'llvm-ranlib'),
    }

    with api.step.nest('zlib'):
      build_zlib(api, lib_install_dir, manifest, env)
    with api.step.nest('libxml2'):
      build_libxml2(api, host_triple, target_triple, lib_install_dir, manifest,
                    env)

    json_manifest = json_format.MessageToJson(
        manifest, preserving_proto_field_name=True)
    api.file.write_text('source manifest', pkg_dir.join('source_manifest.json'),
                        json_manifest)
    api.source_manifest.set_json_manifest('checkout', json_manifest)

    # build clang+llvm
    build_dir = staging_dir.join('llvm_build_dir')
    api.file.ensure_directory('create llvm build dir', build_dir)

    arguments = {
        'cc': cipd_dir.join('bin', 'clang'),
        'cxx': cipd_dir.join('bin', 'clang++'),
        'ar': cipd_dir.join('bin', 'llvm-ar'),
        'ld': cipd_dir.join('bin', 'ld.lld'),
        'nm': cipd_dir.join('bin', 'llvm-nm'),
        'objcopy': cipd_dir.join('bin', 'llvm-objcopy'),
        'objdump': cipd_dir.join('bin', 'llvm-objdump'),
        'ranlib': cipd_dir.join('bin', 'llvm-ranlib'),
        'strip': cipd_dir.join('bin', 'llvm-strip'),
        'gomacc': api.goma.goma_dir.join('gomacc'),
        'ninja': cipd_dir.join('ninja'),
        'target_triple': TRIPLE_TO_NORMALIZED_TRIPLE[target_triple],
        'host_triple': host_triple,
        'target_sysroot': target_sysroot,
        'host_sysroot': host_sysroot,
        'pkgroot_include_dir': lib_install_dir.join('include'),
        'libxml2_include_dir': lib_install_dir.join('include', 'libxml2'),
        'platform_lib_dir': cipd_dir.join(target_platform, 'lib'),
        'pkgroot_lib_dir': lib_install_dir.join('lib'),
        'linux_arm64_sysroot': cipd_dir.join('linux-arm64'),
        'linux_armhf_sysroot': cipd_dir.join('linux-armhf'),
        'linux_i386_sysroot': cipd_dir.join('linux-i386'),
        'linux_amd64_sysroot': cipd_dir.join('linux-amd64'),
        'fuchsia_sdk': cipd_dir.join('sdk'),
    }

    arguments.update({
        'BOOTSTRAP_': 'BOOTSTRAP_',
        'STAGE2_': 'STAGE2_',
    } if prod else {
        'BOOTSTRAP_': '',
        'STAGE2_': '',
    })

    llvm_projects = ['clang', 'clang-tools-extra', 'lld']
    llvm_runtimes = ['compiler-rt', 'libcxx', 'libcxxabi', 'libunwind']

    options = [
        '-GNinja',
        '-DCMAKE_C_COMPILER_LAUNCHER={gomacc}',
        '-DCMAKE_CXX_COMPILER_LAUNCHER={gomacc}',
        '-DCMAKE_ASM_COMPILER_LAUNCHER={gomacc}',
        '-DCMAKE_C_COMPILER={cc}',
        '-DCMAKE_CXX_COMPILER={cxx}',
        '-DCMAKE_ASM_COMPILER={cc}',
        '-DCMAKE_MAKE_PROGRAM={ninja}',
        '-DCMAKE_INSTALL_PREFIX=',
        '-DLLVM_ENABLE_PROJECTS=%s' % ';'.join(llvm_projects),
        '-DLLVM_ENABLE_RUNTIMES=%s' % ';'.join(llvm_runtimes),
    ]

    options.extend({
        'linux': [
            '-DCMAKE_AR={ar}',
            '-DCMAKE_LINKER={ld}',
            '-DCMAKE_NM={nm}',
            '-DCMAKE_OBJCOPY={objcopy}',
            '-DCMAKE_OBJDUMP={objdump}',
            '-DCMAKE_RANLIB={ranlib}',
            '-DCMAKE_STRIP={strip}',
        ],
        'mac': [],
    }[api.platform.name])

    if api.platform.name == 'linux':
      options.extend([
          # BOOTSTRAP_ prefixed flags are passed to the second stage compiler.
          '-D{BOOTSTRAP_}CMAKE_C_FLAGS=-I{pkgroot_include_dir} -I{libxml2_include_dir}',
          '-D{BOOTSTRAP_}CMAKE_CXX_FLAGS=-I{pkgroot_include_dir} -I{libxml2_include_dir}',
          '-D{BOOTSTRAP_}CMAKE_SHARED_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread -L{platform_lib_dir} -L{pkgroot_lib_dir}',
          '-D{BOOTSTRAP_}CMAKE_MODULE_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread -L{platform_lib_dir} -L{pkgroot_lib_dir}',
          '-D{BOOTSTRAP_}CMAKE_EXE_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread -L{platform_lib_dir} -L{pkgroot_lib_dir}',
          '-D{BOOTSTRAP_}CMAKE_SYSROOT={target_sysroot}',
      ])

      if prod:
        options.extend([
            # Unprefixed flags are used by the first stage compiler.
            '-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread',
            '-DCMAKE_MODULE_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread',
            '-DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread',
            '-DCMAKE_SYSROOT={host_sysroot}',
        ])
      else:
        options.extend([
            '-DLLVM_ENABLE_LTO=OFF',
        ])

      if host_triple != target_triple:
        options.extend([
            '-D{BOOTSTRAP_}CMAKE_SYSTEM_NAME=Linux',
            '-D{BOOTSTRAP_}CMAKE_C_COMPILER_TARGET={target_triple}',
            '-D{BOOTSTRAP_}CMAKE_CXX_COMPILER_TARGET={target_triple}',
            '-D{BOOTSTRAP_}LLVM_DEFAULT_TARGET_TRIPLE={target_triple}',
        ])
    elif api.platform.name == 'mac':
      options.extend([
          '-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-nostdlib++ %s' %
          (mode, cipd_dir.join('lib', 'libc++.a'))
          for mode in ['SHARED', 'MODULE', 'EXE']
      ])

      if prod:
        options.extend([
            '-DCMAKE_%s_LINKER_FLAGS=-nostdlib++ %s' %
            (mode, cipd_dir.join('lib', 'libc++.a'))
            for mode in ['SHARED', 'MODULE', 'EXE']
        ])
      else:
        options.extend([
            '-DLLVM_ENABLE_LTO=OFF',
        ])

    # STAGE2_ prefixed flags are passed to the second stage by the first stage build.
    if ci:
      options.extend([
          '-D{STAGE2_}LLVM_ENABLE_ASSERTIONS=ON',
      ])

    options.extend([
        '-D{STAGE2_}LINUX_aarch64-unknown-linux-gnu_SYSROOT={linux_arm64_sysroot}',
        '-D{STAGE2_}LINUX_armv7-unknown-linux-gnueabihf_SYSROOT={linux_armhf_sysroot}',
        '-D{STAGE2_}LINUX_i386-unknown-linux-gnu_SYSROOT={linux_i386_sysroot}',
        '-D{STAGE2_}LINUX_x86_64-unknown-linux-gnu_SYSROOT={linux_amd64_sysroot}',
        '-D{STAGE2_}FUCHSIA_SDK={fuchsia_sdk}',
    ])

    with api.step.nest('clang'), api.context(cwd=build_dir):
      api.step(
          'configure', [cipd_dir.join('bin', 'cmake')] +
          [option.format(**arguments) for option in options] + [
              '-C',
              llvm_dir.join('clang', 'cmake', 'caches', 'Fuchsia%s.cmake' %
                            ('' if prod else '-stage2')),
              llvm_dir.join('llvm')
          ])
      # Build and run tests.
      # TODO: we should be running check-all
      projects = ['llvm', 'clang', 'lld']
      api.step('build', [cipd_dir.join('ninja'), '-j%d' % api.goma.jobs])
      api.step('test', [cipd_dir.join('ninja'),
                        '-j%d' % api.goma.jobs] +
               ['check-%s' % project for project in projects])
      # Build the full (two-stage) distribution.
      target_prefix = 'stage2-' if prod else ''
      api.step(
          'distribution',
          [
              cipd_dir.join('ninja'),
              # This only applies to the first stage, second stage is invoked by
              # CMake as a subprocess and will use Ninja's default.
              '-j%d' % api.goma.jobs,
              target_prefix + 'distribution',
          ])
      with api.context(env={'DESTDIR': pkg_dir}):
        api.step('install', [
            cipd_dir.join('ninja'),
            target_prefix + 'install-distribution-stripped',
        ])

  step_result = api.file.read_text(
      'Version.inc',
      build_dir.join(*(
          (['tools', 'clang', 'stage2-bins'] if prod else []) +
          ['tools', 'clang', 'include', 'clang', 'Basic', 'Version.inc'])),
      test_data='#define CLANG_VERSION_STRING "8.0.0"')
  m = re.search(r'CLANG_VERSION_STRING "([a-zA-Z0-9.-]+)"', step_result)
  assert m, 'Cannot determine Clang version'
  clang_version = m.group(1)

  # TODO(TC-366): Ideally this would be done by the cmake build itself.
  # //zircon/public/gn/toolchain/clang.gni:clang_runtime sets the JSON schema.
  resource_dir_switch = '--resource-dir=clang/%s' % clang_version
  runtime_targets = [[
      '--target',
      '%s-unknown-fuchsia' % arch,
      '--alias',
      '%s-fuchsia' % arch,
  ] for arch in TARGET_TO_ARCH.itervalues()]
  api.python(
      'generate runtimes.json',
      script=api.resource('runtimes.py'),
      args=[
          '--dir', pkg_dir, '--readelf',
          cipd_dir.join('bin', 'llvm-readelf'), resource_dir_switch
      ] + [t for ts in runtime_targets for t in ts],
      stdout=api.json.output(
          leak_to=pkg_dir.join('lib', 'runtime.json'), name='runtime.json'))
  for target in runtime_targets:
    api.python(
        'generate %s.manifest' % target[-1],
        script=api.resource('runtimes.py'),
        args=[
            '--dir', pkg_dir, '--readelf',
            cipd_dir.join('bin', 'llvm-readelf'), resource_dir_switch,
            '--manifest'
        ] + target,
        stdout=api.raw_io.output(
            leak_to=pkg_dir.join('lib', '%s.manifest' % target[-1])))

  api.python(
      'generate license',
      api.resource('generate_license.py'),
      args=['--include'] + [
          llvm_dir.join(project, 'LICENSE.TXT')
          for project in llvm_projects + ['llvm'] + llvm_runtimes
      ] + [
          '--extract',
          api.path['start_dir'].join('libxml2', 'src', 'Copyright'),
          '-',
          '--extract',
          api.path['start_dir'].join('zlib', 'src', 'zlib.h'),
          '4-22',
      ],
      stdout=api.raw_io.output(leak_to=pkg_dir.join('LICENSE')))

  isolated = api.upload.upload_isolated(pkg_dir)

  if prod:
    api.upload.cipd_package(
        'fuchsia/third_party/%s/%s' % (pkg_name, target_platform),
        pkg_dir, [api.upload.DirectoryPath(pkg_dir)],
        {'git_revision': revision},
        repository=repository,
        extra_tags={'version': clang_version})

    host = 'fuchsia.googlesource.com'
    project = 'fuchsia'
    refs = api.gitiles.refs('https://%s/%s' % (host, project))
    ref = 'refs/heads/master'
    revision = refs.get(ref, 'HEAD')

    # Do a full integration build. This will use the just-built toolchain
    # to build all of Fuchsia to check whether there are any regressions.
    api.scheduler.emit_trigger(
        api.scheduler.BuildbucketTrigger(
            properties={
                'clang_toolchain': {
                    'git_repository': repository,
                    'git_revision': revision,
                },
                'build.clang_toolchain': {
                    'type': 'isolated',
                    'instance': isolated,
                },
            },
            tags={
                'buildset':
                    'commit/gitiles/%s/%s/+/%s' % (host, project, revision),
                'gitiles_ref':
                    ref,
            }),
        project='fuchsia',
        jobs={
            'linux-amd64': (
                'fuchsia-arm64-debug-clang',
                'fuchsia-arm64-release-clang',
                'fuchsia-x64-debug-clang',
                'fuchsia-x64-release-clang',
            ),
            'mac-amd64': ('fuchsia-host-mac-clang',),
        }[target_platform])


def GenTests(api):
  for os in ('linux', 'mac'):
    yield (api.test('ci-%s-x64' % os) + api.buildbucket.ci_build(
        project='fuchsia',
        bucket='ci',
        git_repo='https://fuchsia.googlesource.com/third_party/llvm-project',
        revision='a' * 40,
    ) + api.runtime(is_luci=True, is_experimental=False) +
           api.platform.name(os) + api.properties(platform=os + '-amd64'))

    yield (api.test('prod-%s-x64' % os) + api.buildbucket.ci_build(
        project='fuchsia',
        bucket='prod',
        git_repo='https://fuchsia.googlesource.com/third_party/llvm-project',
        revision='a' * 40,
    ) + api.runtime(is_luci=True, is_experimental=False) +
           api.platform.name(os) + api.properties(platform=os + '-amd64') +
           api.gitiles.refs('refs', ('refs/heads/master', 'b' * 40)))

  yield (api.test('linux-arm64') + api.buildbucket.ci_build(
      project='fuchsia',
      bucket='ci',
      git_repo='https://fuchsia.googlesource.com/third_party/llvm-project',
      revision='a' * 40,
  ) + api.runtime(is_luci=True, is_experimental=False) +
         api.platform.name('linux') + api.properties(platform='linux-arm64'))