# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building GCC toolchain."""

from recipe_engine.recipe_api import StepFailure

import re

DEPS = [
    'infra/git',
    'infra/goma',
    'infra/hash',
    'infra/jiri',
    'infra/macos_sdk',
    'infra/upload',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/step',
    'recipe_engine/url',
]

BINUTILS_GIT = 'https://gnu.googlesource.com/binutils-gdb'
BINUTILS_REF = 'refs/tags/binutils-2_32'

GCC_GIT = 'https://gnu.googlesource.com/gcc'
GCC_REF = 'refs/tags/gcc-9_1_0-release'

# Bump this number whenever changing this recipe in ways that affect the
# package built without also changing any upstream revision pin.
# TODO(crbug.com/947158): Remove this when the recipes repo/rev are available.
RECIPE_SALT = '1'


def RunSteps(api):
  api.goma.ensure()

  binutils_dir = api.path['start_dir'].join('binutils-gdb')
  gcc_dir = api.path['start_dir'].join('gcc')

  projects = {
      'binutils-gdb': (BINUTILS_GIT, binutils_dir, BINUTILS_REF),
      'gcc': (GCC_GIT, gcc_dir, GCC_REF),
  }

  gitiles_commit = api.buildbucket.build_input.gitiles_commit
  if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id:
    projects[gitiles_commit.project] = (
        'https://%s/%s' % (gitiles_commit.host, gitiles_commit.project),
        api.path['start_dir'].join(gitiles_commit.project), gitiles_commit.id)

  with api.context(infra_steps=True):
    with api.step.nest('binutils-gdb'):
      binutils_revision = api.git.checkout(*projects['binutils-gdb'])
    with api.step.nest('gcc'):
      gcc_revision = api.git.checkout(*projects['gcc'])

  with api.step.nest('ensure_packages'):
    with api.context(infra_steps=True):
      pkgs = api.cipd.EnsureFile()
      pkgs.add_package('fuchsia/clang/${platform}', 'goma')
      if api.platform.name == 'linux':
        pkgs.add_package('fuchsia/sysroot/${platform}', 'latest')
      cipd_dir = api.path['start_dir'].join('cipd')
      api.cipd.ensure(cipd_dir, pkgs)

  if api.platform.name == 'linux':
    host_sysroot = cipd_dir
  elif api.platform.name == 'mac':
    # TODO(IN-148): Eventually use our own hermetic sysroot as for Linux.
    step_result = api.step(
        'xcrun', ['xcrun', '--show-sdk-path'],
        stdout=api.raw_io.output(name='sdk-path', add_output_log=True),
        step_test_data=lambda: api.raw_io.test_api.stream_output(
            '/some/xcode/path'))
    host_sysroot = step_result.stdout.strip()
  else:  # pragma: no cover
    assert False, "what platform?"

  with api.context(cwd=gcc_dir):
    # download GCC dependencies: GMP, ISL, MPC and MPFR libraries
    api.step('download prerequisites',
             [gcc_dir.join('contrib', 'download_prerequisites')])

  staging_dir = api.path.mkdtemp('gcc')
  pkg_name = 'gcc-%s' % api.platform.name.replace('mac', 'darwin')
  pkg_dir = staging_dir.join(pkg_name)
  api.file.ensure_directory('create pkg dir', pkg_dir)

  # Some of the makefile logic splits $CC at its first word and injects
  # a switch there.  So make $CC and $CXX be single words by writing
  # little scripts.  Autoconf does some checks with CPPFLAGS but not
  # CFLAGS and other checks with CFLAGS but not CPPFLAGS.  The sysroot
  # is necessary for all cases, so fold that into the script too so it's
  # impossible to omit it in any $CC or $CXX invocation.
  cc_path = staging_dir.join('host-cc')
  cxx_path = staging_dir.join('host-cxx')
  for script, compiler in [(cc_path, 'clang'), (cxx_path, 'clang++')]:
    api.file.write_text(
        'write %s script' % api.path.basename(script), script, '''#!/bin/sh
exec %s %s --sysroot=%s "$@"
''' % (api.goma.goma_dir.join('gomacc'), cipd_dir.join('bin',
                                                       compiler), host_sysroot))
    api.step('make %s executable' % api.path.basename(script),
             ['chmod', '+x', script])

  host_cflags = '-O3'
  if api.platform.name != 'mac':
    # LTO works for binutils on Linux but fails on macOS.
    host_cflags += ' -flto'
  host_compiler_args = {
      'CC': '%s' % cc_path,
      'CXX': '%s' % cxx_path,
      'CFLAGS': host_cflags,
      'CXXFLAGS': host_cflags,
  }

  if api.platform.name != 'mac':
    # TODO(mcgrathr): Our host toolchain is currently broken on non-Mac
    # where we provide only static libc++.a that depends on other libraries
    # but we require linking to them explicitly on the command line to
    # satisfy libc++.a's implicit dependencies.
    host_compiler_args['CXXFLAGS'] += ' -lpthread -ldl'

  host_compiler_args = sorted(
      '%s=%s' % item for item in host_compiler_args.iteritems())

  # We force LIMITS_H_TEST=true to tell the compiler to install a <limits.h>
  # that requires a system <limits.h> it can find via #include_next.  But we
  # don't have any system headers for the just-built compiler building target
  # code (i.e. libgcc).  So mock up a little include directory that contains
  # just an empty <limits.h> for it to find.
  mock_include_dir = staging_dir.join('mock-include')
  api.file.ensure_directory('create mock include dir', mock_include_dir)
  api.file.write_text('write dummy <limits.h>',
                      mock_include_dir.join('limits.h'), '/* dummy */')

  with api.macos_sdk(), api.goma.build_with_goma():
    for target, enable_targets in [('aarch64', 'arm-eabi'),
                                   ('x86_64', 'x86_64-pep')]:
      # configure arguments that are the same for binutils and gcc.
      common_args = host_compiler_args + [
          '--prefix=',  # we're building a relocatable package
          '--target=%s-elf' % target,
          '--enable-initfini-array',  # Fuchsia uses .init/.fini arrays
          '--enable-gold',  # Zircon uses gold for userspace build
          # Enable plugins and threading for Gold.
          # This also happens to make it explicitly link in -lpthread and -dl,
          # which are required by host_clang's static libc++.
          '--enable-plugins',
          '--enable-threads',
          '--disable-werror',  # ignore warnings reported by Clang
          '--disable-nls',  # no need for localization
          '--with-included-gettext',  # use include gettext library
      ]

      # build binutils
      binutils_build_dir = staging_dir.join('binutils_%s_build_dir' % target)
      api.file.ensure_directory('create binutils %s build dir' % target,
                                binutils_build_dir)

      with api.context(cwd=binutils_build_dir):

        def binutils_make_step(name, make_target, prefix, jobs, make_args=None):
          # As of 2.32, gold/testsuite/Makefile.am unconditionally edits in
          # a -B.../ switch to make the compiler use the just-built gold as
          # the linker for test suite binaries.  This is wrong when
          # building a cross-linker.  Force it to a no-op on the make
          # command line to work around the bug.
          # TODO(mcgrathr): Drop this when we roll to a binutils that has
          # this fixed upstream.
          make_args = make_args or []
          make_args.append('MAKEOVERRIDES=editcc=-eb')
          return api.step(
              '%s %s binutils' % (name, make_target),
              ['make', '-j%s' % jobs] + make_args + [
                  '%s-%s' % (prefix, component)
                  for component in ['binutils', 'gas', 'ld', 'gold']
              ])

        api.step(
            'configure %s binutils' % target,
            [
                binutils_dir.join('configure'),
                '--enable-deterministic-archives',  # more deterministic builds
                '--enable-targets=%s' % enable_targets,
            ] + common_args)
        binutils_make_step('build', target, 'all', api.goma.jobs)
        try:
          binutils_make_step('test', target, 'check', api.platform.cpu_count,
                             ['-k'])
        except StepFailure as error:
          logs = {
              l[0]:
              api.file.read_text('binutils %s %s' % (target, '/'.join(l)),
                                 binutils_build_dir.join(*l)).splitlines()
              for l in [
                  ('gas', 'testsuite', 'gas.log'),
                  ('binutils', 'binutils.log'),
                  ('ld', 'ld.log'),
                  ('gold', 'testsuite', 'test-suite.log'),
              ]
          }
          step_result = api.step('binutils test failure', cmd=None)
          for name, text in logs.iteritems():
            step_result.presentation.logs[name] = text
          raise error
        binutils_make_step('install', target, 'install-strip', 1,
                           ['DESTDIR=%s' % pkg_dir])

      # build gcc
      gcc_build_dir = staging_dir.join('gcc_%s_build_dir' % target)
      api.file.ensure_directory('create gcc %s build dir' % target,
                                gcc_build_dir)

      with api.context(
          cwd=gcc_build_dir, env_prefixes={'PATH': [pkg_dir.join('bin')]}):
        gcc_goals = ['gcc', 'target-libgcc']

        def gcc_make_step(name, make_target, jobs, args):
          cmd = [
              'make',
              '-j%s' % jobs,
              'MAKEOVERRIDES=USE_GCC_STDINT=provide LIMITS_H_TEST=true',
              # Force flags for libgcc to get <limits.h> kludge.
              'CPPFLAGS_FOR_TARGET=-idirafter %s' % mock_include_dir,
          ]
          return api.step('%s %s gcc' % (name, make_target), cmd + args)

        api.step(
            'configure %s gcc' % target,
            [
                gcc_dir.join('configure'),
                '--enable-languages=c,c++',
                '--disable-libstdcxx',  # we don't need libstdc++
                '--disable-libssp',  # we don't need libssp either
                '--disable-libquadmath',  # and nor do we need libquadmath
            ] + common_args)
        gcc_make_step('build', target, api.goma.jobs,
                      ['all-%s' % goal for goal in gcc_goals])
        try:
          gcc_make_step('test', target, api.platform.cpu_count, ['check-gcc'])
        finally:
          logs = {
              l[-1]: api.file.read_text('gcc %s %s' % (target, '/'.join(l)),
                                        gcc_build_dir.join(*l)).splitlines()
              for l in [
                  ('gcc', 'testsuite', 'gcc', 'gcc.log'),
                  ('gcc', 'testsuite', 'g++', 'g++.log'),
              ]
          }
          step_result = api.step('test %s gcc logs' % target, cmd=None)
          for name, text in logs.iteritems():
            step_result.presentation.logs[name] = text
        gcc_make_step('install', target, 1, ['DESTDIR=%s' % pkg_dir] +
                      ['install-strip-%s' % goal for goal in gcc_goals])

  binutils_version = api.file.read_text(
      'binutils version',
      binutils_dir.join('bfd', 'version.m4'),
      test_data='m4_define([BFD_VERSION], [2.27.0])')
  m = re.match(r'm4_define\(\[BFD_VERSION\], \[([^]]+)\]\)', binutils_version)
  assert m and m.group(1), ('bfd/version.m4 has unexpected format: %r' %
                            binutils_version)
  binutils_version = m.group(1)
  gcc_version = api.file.read_text('gcc version',
                                   gcc_dir.join('gcc', 'BASE-VER')).rstrip()

  cipd_git_repository = ','.join([GCC_GIT, BINUTILS_GIT])
  cipd_git_revision = ','.join([gcc_revision, binutils_revision])

  # Add a bogus "salt" repository/revision to represent recipe changes when
  # the upstream revisions haven't changed.  Bump this number whenever
  # changing this recipe in ways that affect the package built without also
  # changing any upstream revision pin.
  # TODO(crbug.com/947158): Replace this with the recipes repo/rev when
  # infra makes that available here.
  if RECIPE_SALT:  # pragma: no cover
    cipd_git_repository += ',<salt>'
    cipd_git_revision += ',%s' % RECIPE_SALT

  version = ','.join([gcc_version, binutils_version])
  api.upload.cipd_package(
      'fuchsia/gcc/${platform}',
      pkg_dir, [api.upload.DirectoryPath(pkg_dir)],
      {'git_revision': cipd_git_revision},
      repository=cipd_git_repository,
      extra_tags={'version': version})


def GenTests(api):
  binutils_revision = '3d861fdb826c2f5cf270dd5f585d0e6057e1bf4f'
  gcc_revision = '4b5e15daff8b54440e3fda451c318ad31e532fab'
  cipd_revision = ','.join([gcc_revision, binutils_revision])
  if RECIPE_SALT:  # pragma: no cover
    cipd_revision += ',%s' % RECIPE_SALT

  for platform in ('linux', 'mac'):
    yield (api.test(platform) + api.platform.name(platform) +
           api.step_data('binutils-gdb.git rev-parse',
                         api.raw_io.stream_output(binutils_revision)) +
           api.step_data('gcc.git rev-parse',
                         api.raw_io.stream_output(gcc_revision)))
    for salt in ['', 999] + ([RECIPE_SALT] if RECIPE_SALT else []):
      salt = '%s' % salt
      yield (api.test(platform + '_new' + salt) + api.platform.name(platform) +
             api.step_data('binutils-gdb.git rev-parse',
                           api.raw_io.stream_output(binutils_revision)) +
             api.step_data('gcc.git rev-parse',
                           api.raw_io.stream_output(gcc_revision)) +
             api.step_data(
                 'cipd search fuchsia/gcc/${platform} git_revision:' +
                 cipd_revision, api.json.output({'result': []})) +
             api.step_data('gcc version', api.file.read_text('7.1.2\n')))
      yield (api.test(platform + '_binutils_test_fail' + salt) +
             api.buildbucket.ci_build(
                 git_repo=BINUTILS_GIT, revision=binutils_revision) +
             api.step_data('gcc.git rev-parse',
                           api.raw_io.stream_output(gcc_revision)) +
             api.platform.name(platform) +
             api.step_data('test x86_64 binutils', retcode=1))
      yield (api.test(platform + '_gcc_test_fail' + salt) +
             api.buildbucket.ci_build(git_repo=GCC_GIT, revision=gcc_revision) +
             api.step_data('binutils-gdb.git rev-parse',
                           api.raw_io.stream_output(binutils_revision)) +
             api.platform.name(platform) +
             api.step_data('test aarch64 gcc', retcode=1))