recipes/gcc_toolchain.py - infra/recipes - Git at Google

 # Copyright 2018 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """Recipe for building GCC toolchain."""

 from recipe_engine.config import Enum, ReturnSchema, Single
 from recipe_engine.recipe_api import Property, StepFailure

 import re


 DEPS = [
   'infra/git',
   'infra/gitiles',
   'infra/goma',
   'infra/hash',
   'infra/jiri',
   'recipe_engine/cipd',
   'recipe_engine/context',
   'recipe_engine/file',
   'recipe_engine/json',
   'recipe_engine/path',
   'recipe_engine/platform',
   'recipe_engine/properties',
   'recipe_engine/raw_io',
   'recipe_engine/step',
   'recipe_engine/tempfile',
   'recipe_engine/url',
 ]

 BINUTILS_GIT = 'https://gnu.googlesource.com/binutils-gdb'
 BINUTILS_REF = 'refs/heads/binutils-2_31-branch'

 GCC_GIT = 'https://gnu.googlesource.com/gcc'
 GCC_REF = 'refs/tags/gcc-8_2_0-release'

 PROPERTIES = {
   'binutils_revision': Property(kind=str, help='Revision in binutils repo',
                                 default=None),
   'gcc_revision': Property(kind=str, help='Revision in GCC repo', default=None),
 }


 # binutils and gcc both have a daily robocommit to a specific file that
 # just updates a date stamp.  If that's the only change since the last
 # revision we built, this new revision is not worth building.
 def DoCheckout(api, url, checkout_dir, revision, last_revision, useless_file):
   api.git.checkout(url, checkout_dir, revision)
   with api.context(cwd=checkout_dir):
     try:
       step = api.git(
           'diff', '--name-only', '%s..%s' % (last_revision, revision),
           name='check for changes other than %s' % useless_file,
           stdout=api.raw_io.output(name='changed files', add_output_log=True))
     except StepFailure as error:
       # If the old revision is no longer in the repo or something like that,
       # git can fail.  But that shouldn't make the build fail, it should just
       # not trigger the "useless change" short-circuit.
       return False
   changed_files = set(step.stdout.splitlines()) - set([useless_file])
   return changed_files == set()


 def RunSteps(api, binutils_revision, gcc_revision):
   api.gitiles.ensure_gitiles()
   api.goma.ensure_goma()

   if binutils_revision is None:
     binutils_revision = api.gitiles.refs(
         BINUTILS_GIT, step_name='binutils refs').get(BINUTILS_REF, None)
   if gcc_revision is None:
     gcc_revision = api.gitiles.refs(
         GCC_GIT, step_name='gcc refs').get(GCC_REF, None)

   cipd_pkg_name = 'fuchsia/gcc/${platform}'
   cipd_git_revision = ','.join([gcc_revision, binutils_revision])

   pins = api.cipd.search(cipd_pkg_name, 'git_revision:' + cipd_git_revision)
   if len(pins) > 0:
     api.step('Package is up-to-date', cmd=None)
     return

   with api.step.nest('ensure_packages'):
     with api.context(infra_steps=True):
       pkgs = api.cipd.EnsureFile()
       pkgs.add_package('fuchsia/clang/${platform}', 'goma')
       if api.platform.name == 'linux':
         pkgs.add_package('fuchsia/sysroot/${platform}', 'latest')
       cipd_dir = api.path['start_dir'].join('cipd')
       api.cipd.ensure(cipd_dir, pkgs)

   if api.platform.name == 'linux':
     host_sysroot = cipd_dir
   elif api.platform.name == 'mac':
     # TODO(IN-148): Eventually use our own hermetic sysroot as for Linux.
     step_result = api.step(
         'xcrun', ['xcrun', '--show-sdk-path'],
         stdout=api.raw_io.output(name='sdk-path', add_output_log=True),
         step_test_data=lambda: api.raw_io.test_api.stream_output(
             '/some/xcode/path'))
     host_sysroot = step_result.stdout.strip()
   else: # pragma: no cover
     assert false, "what platform?"

   description = api.cipd.describe(
       cipd_pkg_name, 'latest', test_data_tags=[
         'git_revision:b824b5a2484202f6eabbe118b1cf7682ce77b76e,5898623efb820fc56123da2e316756a9a32cb59b'])

   [last_tag] = [cipd_tag.tag
                 for cipd_tag in description.tags
                 if cipd_tag.tag.startswith('git_revision:')]
   [last_gcc_revision, last_binutils_revision] = last_tag[
       len('git_revision:'):].split(',')

   with api.context(infra_steps=True):
     binutils_dir = api.path['start_dir'].join('binutils-gdb')
     binutils_no_change = DoCheckout(api, BINUTILS_GIT, binutils_dir,
                                     binutils_revision, last_binutils_revision,
                                     'bfd/version.h')
     gcc_dir = api.path['start_dir'].join('gcc')
     gcc_no_change = DoCheckout(api, GCC_GIT, gcc_dir,
                                gcc_revision, last_gcc_revision,
                                'gcc/DATESTAMP')
     if binutils_no_change and gcc_no_change:
       api.step('Revisions contain no useful changes', cmd=None)
       return

   with api.context(cwd=gcc_dir):
     # download GCC dependencies: GMP, ISL, MPC and MPFR libraries
     api.step('download prerequisites', [
       gcc_dir.join('contrib', 'download_prerequisites')
     ])

   staging_dir = api.path.mkdtemp('gcc')
   pkg_name = 'gcc-%s' % api.platform.name.replace('mac', 'darwin')
   pkg_dir = staging_dir.join(pkg_name)
   api.file.ensure_directory('create pkg dir', pkg_dir)

   host_clang = '%s %s' % (api.goma.goma_dir.join('gomacc'),
                           cipd_dir.join('bin', 'clang'))
   host_cflags = '-O3 --sysroot=%s' % host_sysroot
   if api.platform.name != 'mac':
     # LTO works for binutils on Linux but fails on macOS.
     host_cflags += ' -flto'
   host_compiler_args = {
       'CC': host_clang,
       'CXX': '%s++' % host_clang,
       'CFLAGS': host_cflags,
       'CXXFLAGS': host_cflags,
   }

   if api.platform.name != 'mac':
     # TODO(mcgrathr): Our host toolchain is currently broken on non-Mac
     # where we provide only static libc++.a that depends on other libraries
     # but we require linking to them explicitly on the command line to
     # satisfy libc++.a's implicit dependencies.
     host_compiler_args['CXXFLAGS'] += ' -lpthread -ldl'

   host_compiler_args = sorted('%s=%s' % item
                               for item in host_compiler_args.iteritems())

   with api.goma.build_with_goma():
     for target, enable_targets in [('aarch64', 'arm-eabi'),
                                    ('x86_64', 'x86_64-pep')]:
       # configure arguments that are the same for binutils and gcc.
       common_args =  host_compiler_args + [
           '--prefix=', # we're building a relocatable package
           '--target=%s-elf' % target,
           '--enable-initfini-array', # Fuchsia uses .init/.fini arrays
           '--enable-gold', # Zircon uses gold for userspace build
           # Enable plugins and threading for Gold.
           # This also happens to make it explicitly link in -lpthread and -dl,
           # which are required by host_clang's static libc++.
           '--enable-plugins',
           '--enable-threads',
           '--disable-werror', # ignore warnings reported by Clang
           '--disable-nls', # no need for localization
           '--with-included-gettext', # use include gettext library
       ]

       # build binutils
       binutils_build_dir = staging_dir.join('binutils_%s_build_dir' % target)
       api.file.ensure_directory('create binutils %s build dir' % target,
                                 binutils_build_dir)

       with api.context(cwd=binutils_build_dir):
         def binutils_make_step(name, prefix, jobs, make_args=[], logs=[]):
           return api.step('%s %s binutils' % (name, target),
                           ['make','-j%s' % jobs] + make_args +
                           ['%s-%s' % (prefix, component)
                            for component in ['binutils', 'gas', 'ld', 'gold']])
         api.step('configure %s binutils' % target, [
           binutils_dir.join('configure'),
           '--enable-deterministic-archives', # more deterministic builds
           '--enable-targets=%s' % enable_targets,
         ] + common_args)
         binutils_make_step('build', 'all', api.goma.jobs)
         try:
           binutils_make_step('test', 'check', api.platform.cpu_count, ['-k'])
         except StepFailure as error:
           logs = {
             l[0]: api.file.read_text('binutils %s %s' % (target, '/'.join(l)),
                                      binutils_build_dir.join(*l)).splitlines()
             for l in [
               ('gas', 'testsuite', 'gas.log'),
               ('binutils', 'binutils.log'),
               ('ld', 'ld.log'),
               ('gold', 'testsuite', 'test-suite.log'),
             ]
           }
           step_result = api.step('binutils test failure', cmd=None)
           for name, text in logs.iteritems():
             step_result.presentation.logs[name] = text
           raise error
         binutils_make_step('install', 'install-strip', 1,
                            ['DESTDIR=%s' % pkg_dir])

       # build gcc
       gcc_build_dir = staging_dir.join('gcc_%s_build_dir' % target)
       api.file.ensure_directory('create gcc %s build dir' % target,
                                 gcc_build_dir)

       with api.context(cwd=gcc_build_dir,
                        env_prefixes={'PATH': [pkg_dir.join('bin')]}):
         gcc_goals = ['gcc', 'target-libgcc']
         def gcc_make_step(name, jobs, args):
           cmd = [
               'make',
               '-j%s' % jobs,
               'MAKEOVERRIDES=USE_GCC_STDINT=provide',
           ]
           return api.step('%s %s gcc' % (name, target), cmd + args)
         api.step('configure %s gcc' % target, [
           gcc_dir.join('configure'),
           '--enable-languages=c,c++',
           '--disable-libstdcxx', # we don't need libstdc++
           '--disable-libssp', # we don't need libssp either
           '--disable-libquadmath', # and nor do we need libquadmath
         ] + common_args)
         gcc_make_step('build', api.goma.jobs,
                       ['all-%s' % goal for goal in gcc_goals])
         try:
           gcc_make_step('test', api.platform.cpu_count, ['check-gcc'])
         finally:
           logs = {
             l[-1]: api.file.read_text('gcc %s %s' % (target, '/'.join(l)),
                                       gcc_build_dir.join(*l)).splitlines()
             for l in [
               ('gcc', 'testsuite', 'gcc', 'gcc.log'),
               ('gcc', 'testsuite', 'g++', 'g++.log'),
             ]
           }
           step_result = api.step('test %s gcc logs' % target, cmd=None)
           for name, text in logs.iteritems():
             step_result.presentation.logs[name] = text
         gcc_make_step('install', 1,
                       ['DESTDIR=%s' % pkg_dir] +
                       ['install-strip-%s' % goal for goal in gcc_goals])

   binutils_version = api.file.read_text('binutils version',
                                         binutils_dir.join('bfd', 'version.m4'))
   m = re.match(r'm4_define\(\[BFD_VERSION\], \[([^]]+)\]\)', binutils_version)
   assert m and m.group(1), 'bfd/version.m4 has unexpected format'
   binutils_version = m.group(1)
   gcc_version = api.file.read_text('gcc version',
                                    gcc_dir.join('gcc', 'BASE-VER')).rstrip()

   pkg_def = api.cipd.PackageDefinition(
       package_name=cipd_pkg_name,
       package_root=pkg_dir,
       install_mode='copy')
   pkg_def.add_dir(pkg_dir)
   pkg_def.add_version_file('.versions/gcc.cipd_version')

   cipd_pkg_file = api.path['cleanup'].join('gcc.cipd')

   api.cipd.build_from_pkg(
       pkg_def=pkg_def,
       output_package=cipd_pkg_file,
   )
   step_result = api.cipd.register(
       package_name=cipd_pkg_name,
       package_path=cipd_pkg_file,
       refs=['latest'],
       tags={
         'version': ','.join([gcc_version, binutils_version]),
         'git_repository': ','.join([GCC_GIT, BINUTILS_GIT]),
         'git_revision': cipd_git_revision,
       },
   )


 def GenTests(api):
   binutils_revision = '3d861fdb826c2f5cf270dd5f585d0e6057e1bf4f'
   gcc_revision = '4b5e15daff8b54440e3fda451c318ad31e532fab'
   cipd_revision = ','.join([gcc_revision, binutils_revision])
   for platform in ('linux', 'mac'):
     yield (api.test(platform) +
            api.platform.name(platform) +
            api.gitiles.refs('binutils refs',
                             (BINUTILS_REF, binutils_revision)) +
            api.gitiles.refs('gcc refs',
                             (GCC_REF, gcc_revision)))
     yield (api.test(platform + '_new') +
            api.platform.name(platform) +
            api.gitiles.refs('binutils refs',
                             (BINUTILS_REF, binutils_revision)) +
            api.gitiles.refs('gcc refs',
                             (GCC_REF, gcc_revision)) +
            api.step_data('cipd search fuchsia/gcc/${platform} ' +
                          'git_revision:' + cipd_revision,
                          api.json.output({'result': []})) +
            api.step_data('check for changes other than bfd/version.h',
                          api.raw_io.stream_output('bfd/version.h\nothers\n',
                                                   name='changed files')) +
            api.step_data('check for changes other than gcc/DATESTAMP',
                          api.raw_io.stream_output('gcc/DATESTAMP\nothers\n',
                                                   name='changed files')) +
            api.step_data('binutils version', api.file.read_text(
                'm4_define([BFD_VERSION], [2.27.0])')) +
            api.step_data('gcc version', api.file.read_text('7.1.2\n')))
     yield (api.test(platform + '_binutils_test_fail') +
            api.platform.name(platform) +
            api.gitiles.refs('binutils refs',
                             (BINUTILS_REF, binutils_revision)) +
            api.gitiles.refs('gcc refs',
                             (GCC_REF, gcc_revision)) +
            api.step_data('cipd search fuchsia/gcc/${platform} ' +
                          'git_revision:' + cipd_revision,
                          api.json.output({'result': []})) +
            api.step_data('check for changes other than bfd/version.h',
                          api.raw_io.stream_output('bfd/version.h\nothers\n',
                                                   name='changed files')) +
            api.step_data('check for changes other than gcc/DATESTAMP',
                          api.raw_io.stream_output('gcc/DATESTAMP\nothers\n',
                                                   name='changed files')) +
            api.step_data('test x86_64 binutils', retcode=1))
     yield (api.test(platform + '_gcc_test_fail') +
            api.platform.name(platform) +
            api.gitiles.refs('binutils refs',
                             (BINUTILS_REF, binutils_revision)) +
            api.gitiles.refs('gcc refs',
                             (GCC_REF, gcc_revision)) +
            api.step_data('cipd search fuchsia/gcc/${platform} ' +
                          'git_revision:' + cipd_revision,
                          api.json.output({'result': []})) +
            api.step_data('check for changes other than bfd/version.h',
                          api.raw_io.stream_output('bfd/version.h\nothers\n',
                                                   name='changed files')) +
            api.step_data('check for changes other than gcc/DATESTAMP',
                          api.raw_io.stream_output('', retcode=1,
                                                   name='changed files')) +
            api.step_data('test aarch64 gcc', retcode=1))
     yield (api.test(platform + '_useless') +
            api.platform.name(platform) +
            api.gitiles.refs('binutils refs',
                             (BINUTILS_REF, binutils_revision)) +
            api.gitiles.refs('gcc refs',
                             (GCC_REF, gcc_revision)) +
            api.step_data('cipd search fuchsia/gcc/${platform} ' +
                          'git_revision:' + cipd_revision,
                          api.json.output({'result': []})) +
            api.step_data('check for changes other than bfd/version.h',
                          api.raw_io.stream_output('bfd/version.h\n',
                                                   name='changed files')) +
            api.step_data('check for changes other than gcc/DATESTAMP',
                          api.raw_io.stream_output('gcc/DATESTAMP\n',
                                                   name='changed files')))
	# Copyright 2018 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""Recipe for building GCC toolchain."""

	from recipe_engine.config import Enum, ReturnSchema, Single
	from recipe_engine.recipe_api import Property, StepFailure

	import re


	DEPS = [
	'infra/git',
	'infra/gitiles',
	'infra/goma',
	'infra/hash',
	'infra/jiri',
	'recipe_engine/cipd',
	'recipe_engine/context',
	'recipe_engine/file',
	'recipe_engine/json',
	'recipe_engine/path',
	'recipe_engine/platform',
	'recipe_engine/properties',
	'recipe_engine/raw_io',
	'recipe_engine/step',
	'recipe_engine/tempfile',
	'recipe_engine/url',
	]

	BINUTILS_GIT = 'https://gnu.googlesource.com/binutils-gdb'
	BINUTILS_REF = 'refs/heads/binutils-2_31-branch'

	GCC_GIT = 'https://gnu.googlesource.com/gcc'
	GCC_REF = 'refs/tags/gcc-8_2_0-release'

	PROPERTIES = {
	'binutils_revision': Property(kind=str, help='Revision in binutils repo',
	default=None),
	'gcc_revision': Property(kind=str, help='Revision in GCC repo', default=None),
	}


	# binutils and gcc both have a daily robocommit to a specific file that
	# just updates a date stamp. If that's the only change since the last
	# revision we built, this new revision is not worth building.
	def DoCheckout(api, url, checkout_dir, revision, last_revision, useless_file):
	api.git.checkout(url, checkout_dir, revision)
	with api.context(cwd=checkout_dir):
	try:
	step = api.git(
	'diff', '--name-only', '%s..%s' % (last_revision, revision),
	name='check for changes other than %s' % useless_file,
	stdout=api.raw_io.output(name='changed files', add_output_log=True))
	except StepFailure as error:
	# If the old revision is no longer in the repo or something like that,
	# git can fail. But that shouldn't make the build fail, it should just
	# not trigger the "useless change" short-circuit.
	return False
	changed_files = set(step.stdout.splitlines()) - set([useless_file])
	return changed_files == set()


	def RunSteps(api, binutils_revision, gcc_revision):
	api.gitiles.ensure_gitiles()
	api.goma.ensure_goma()

	if binutils_revision is None:
	binutils_revision = api.gitiles.refs(
	BINUTILS_GIT, step_name='binutils refs').get(BINUTILS_REF, None)
	if gcc_revision is None:
	gcc_revision = api.gitiles.refs(
	GCC_GIT, step_name='gcc refs').get(GCC_REF, None)

	cipd_pkg_name = 'fuchsia/gcc/${platform}'
	cipd_git_revision = ','.join([gcc_revision, binutils_revision])

	pins = api.cipd.search(cipd_pkg_name, 'git_revision:' + cipd_git_revision)
	if len(pins) > 0:
	api.step('Package is up-to-date', cmd=None)
	return

	with api.step.nest('ensure_packages'):
	with api.context(infra_steps=True):
	pkgs = api.cipd.EnsureFile()
	pkgs.add_package('fuchsia/clang/${platform}', 'goma')
	if api.platform.name == 'linux':
	pkgs.add_package('fuchsia/sysroot/${platform}', 'latest')
	cipd_dir = api.path['start_dir'].join('cipd')
	api.cipd.ensure(cipd_dir, pkgs)

	if api.platform.name == 'linux':
	host_sysroot = cipd_dir
	elif api.platform.name == 'mac':
	# TODO(IN-148): Eventually use our own hermetic sysroot as for Linux.
	step_result = api.step(
	'xcrun', ['xcrun', '--show-sdk-path'],
	stdout=api.raw_io.output(name='sdk-path', add_output_log=True),
	step_test_data=lambda: api.raw_io.test_api.stream_output(
	'/some/xcode/path'))
	host_sysroot = step_result.stdout.strip()
	else: # pragma: no cover
	assert false, "what platform?"

	description = api.cipd.describe(
	cipd_pkg_name, 'latest', test_data_tags=[
	'git_revision:b824b5a2484202f6eabbe118b1cf7682ce77b76e,5898623efb820fc56123da2e316756a9a32cb59b'])

	[last_tag] = [cipd_tag.tag
	for cipd_tag in description.tags
	if cipd_tag.tag.startswith('git_revision:')]
	[last_gcc_revision, last_binutils_revision] = last_tag[
	len('git_revision:'):].split(',')

	with api.context(infra_steps=True):
	binutils_dir = api.path['start_dir'].join('binutils-gdb')
	binutils_no_change = DoCheckout(api, BINUTILS_GIT, binutils_dir,
	binutils_revision, last_binutils_revision,
	'bfd/version.h')
	gcc_dir = api.path['start_dir'].join('gcc')
	gcc_no_change = DoCheckout(api, GCC_GIT, gcc_dir,
	gcc_revision, last_gcc_revision,
	'gcc/DATESTAMP')
	if binutils_no_change and gcc_no_change:
	api.step('Revisions contain no useful changes', cmd=None)
	return

	with api.context(cwd=gcc_dir):
	# download GCC dependencies: GMP, ISL, MPC and MPFR libraries
	api.step('download prerequisites', [
	gcc_dir.join('contrib', 'download_prerequisites')
	])

	staging_dir = api.path.mkdtemp('gcc')
	pkg_name = 'gcc-%s' % api.platform.name.replace('mac', 'darwin')
	pkg_dir = staging_dir.join(pkg_name)
	api.file.ensure_directory('create pkg dir', pkg_dir)

	host_clang = '%s %s' % (api.goma.goma_dir.join('gomacc'),
	cipd_dir.join('bin', 'clang'))
	host_cflags = '-O3 --sysroot=%s' % host_sysroot
	if api.platform.name != 'mac':
	# LTO works for binutils on Linux but fails on macOS.
	host_cflags += ' -flto'
	host_compiler_args = {
	'CC': host_clang,
	'CXX': '%s++' % host_clang,
	'CFLAGS': host_cflags,
	'CXXFLAGS': host_cflags,
	}

	if api.platform.name != 'mac':
	# TODO(mcgrathr): Our host toolchain is currently broken on non-Mac
	# where we provide only static libc++.a that depends on other libraries
	# but we require linking to them explicitly on the command line to
	# satisfy libc++.a's implicit dependencies.
	host_compiler_args['CXXFLAGS'] += ' -lpthread -ldl'

	host_compiler_args = sorted('%s=%s' % item
	for item in host_compiler_args.iteritems())

	with api.goma.build_with_goma():
	for target, enable_targets in [('aarch64', 'arm-eabi'),
	('x86_64', 'x86_64-pep')]:
	# configure arguments that are the same for binutils and gcc.
	common_args = host_compiler_args + [
	'--prefix=', # we're building a relocatable package
	'--target=%s-elf' % target,
	'--enable-initfini-array', # Fuchsia uses .init/.fini arrays
	'--enable-gold', # Zircon uses gold for userspace build
	# Enable plugins and threading for Gold.
	# This also happens to make it explicitly link in -lpthread and -dl,
	# which are required by host_clang's static libc++.
	'--enable-plugins',
	'--enable-threads',
	'--disable-werror', # ignore warnings reported by Clang
	'--disable-nls', # no need for localization
	'--with-included-gettext', # use include gettext library
	]

	# build binutils
	binutils_build_dir = staging_dir.join('binutils_%s_build_dir' % target)
	api.file.ensure_directory('create binutils %s build dir' % target,
	binutils_build_dir)

	with api.context(cwd=binutils_build_dir):
	def binutils_make_step(name, prefix, jobs, make_args=[], logs=[]):
	return api.step('%s %s binutils' % (name, target),
	['make','-j%s' % jobs] + make_args +
	['%s-%s' % (prefix, component)
	for component in ['binutils', 'gas', 'ld', 'gold']])
	api.step('configure %s binutils' % target, [
	binutils_dir.join('configure'),
	'--enable-deterministic-archives', # more deterministic builds
	'--enable-targets=%s' % enable_targets,
	] + common_args)
	binutils_make_step('build', 'all', api.goma.jobs)
	try:
	binutils_make_step('test', 'check', api.platform.cpu_count, ['-k'])
	except StepFailure as error:
	logs = {
	l[0]: api.file.read_text('binutils %s %s' % (target, '/'.join(l)),
	binutils_build_dir.join(*l)).splitlines()
	for l in [
	('gas', 'testsuite', 'gas.log'),
	('binutils', 'binutils.log'),
	('ld', 'ld.log'),
	('gold', 'testsuite', 'test-suite.log'),
	]
	}
	step_result = api.step('binutils test failure', cmd=None)
	for name, text in logs.iteritems():
	step_result.presentation.logs[name] = text
	raise error
	binutils_make_step('install', 'install-strip', 1,
	['DESTDIR=%s' % pkg_dir])

	# build gcc
	gcc_build_dir = staging_dir.join('gcc_%s_build_dir' % target)
	api.file.ensure_directory('create gcc %s build dir' % target,
	gcc_build_dir)

	with api.context(cwd=gcc_build_dir,
	env_prefixes={'PATH': [pkg_dir.join('bin')]}):
	gcc_goals = ['gcc', 'target-libgcc']
	def gcc_make_step(name, jobs, args):
	cmd = [
	'make',
	'-j%s' % jobs,
	'MAKEOVERRIDES=USE_GCC_STDINT=provide',
	]
	return api.step('%s %s gcc' % (name, target), cmd + args)
	api.step('configure %s gcc' % target, [
	gcc_dir.join('configure'),
	'--enable-languages=c,c++',
	'--disable-libstdcxx', # we don't need libstdc++
	'--disable-libssp', # we don't need libssp either
	'--disable-libquadmath', # and nor do we need libquadmath
	] + common_args)
	gcc_make_step('build', api.goma.jobs,
	['all-%s' % goal for goal in gcc_goals])
	try:
	gcc_make_step('test', api.platform.cpu_count, ['check-gcc'])
	finally:
	logs = {
	l[-1]: api.file.read_text('gcc %s %s' % (target, '/'.join(l)),
	gcc_build_dir.join(*l)).splitlines()
	for l in [
	('gcc', 'testsuite', 'gcc', 'gcc.log'),
	('gcc', 'testsuite', 'g++', 'g++.log'),
	]
	}
	step_result = api.step('test %s gcc logs' % target, cmd=None)
	for name, text in logs.iteritems():
	step_result.presentation.logs[name] = text
	gcc_make_step('install', 1,
	['DESTDIR=%s' % pkg_dir] +
	['install-strip-%s' % goal for goal in gcc_goals])

	binutils_version = api.file.read_text('binutils version',
	binutils_dir.join('bfd', 'version.m4'))
	m = re.match(r'm4_define\(\[BFD_VERSION\], \[([^]]+)\]\)', binutils_version)
	assert m and m.group(1), 'bfd/version.m4 has unexpected format'
	binutils_version = m.group(1)
	gcc_version = api.file.read_text('gcc version',
	gcc_dir.join('gcc', 'BASE-VER')).rstrip()

	pkg_def = api.cipd.PackageDefinition(
	package_name=cipd_pkg_name,
	package_root=pkg_dir,
	install_mode='copy')
	pkg_def.add_dir(pkg_dir)
	pkg_def.add_version_file('.versions/gcc.cipd_version')

	cipd_pkg_file = api.path['cleanup'].join('gcc.cipd')

	api.cipd.build_from_pkg(
	pkg_def=pkg_def,
	output_package=cipd_pkg_file,
	)
	step_result = api.cipd.register(
	package_name=cipd_pkg_name,
	package_path=cipd_pkg_file,
	refs=['latest'],
	tags={
	'version': ','.join([gcc_version, binutils_version]),
	'git_repository': ','.join([GCC_GIT, BINUTILS_GIT]),
	'git_revision': cipd_git_revision,
	},
	)


	def GenTests(api):
	binutils_revision = '3d861fdb826c2f5cf270dd5f585d0e6057e1bf4f'
	gcc_revision = '4b5e15daff8b54440e3fda451c318ad31e532fab'
	cipd_revision = ','.join([gcc_revision, binutils_revision])
	for platform in ('linux', 'mac'):
	yield (api.test(platform) +
	api.platform.name(platform) +
	api.gitiles.refs('binutils refs',
	(BINUTILS_REF, binutils_revision)) +
	api.gitiles.refs('gcc refs',
	(GCC_REF, gcc_revision)))
	yield (api.test(platform + '_new') +
	api.platform.name(platform) +
	api.gitiles.refs('binutils refs',
	(BINUTILS_REF, binutils_revision)) +
	api.gitiles.refs('gcc refs',
	(GCC_REF, gcc_revision)) +
	api.step_data('cipd search fuchsia/gcc/${platform} ' +
	'git_revision:' + cipd_revision,
	api.json.output({'result': []})) +
	api.step_data('check for changes other than bfd/version.h',
	api.raw_io.stream_output('bfd/version.h\nothers\n',
	name='changed files')) +
	api.step_data('check for changes other than gcc/DATESTAMP',
	api.raw_io.stream_output('gcc/DATESTAMP\nothers\n',
	name='changed files')) +
	api.step_data('binutils version', api.file.read_text(
	'm4_define([BFD_VERSION], [2.27.0])')) +
	api.step_data('gcc version', api.file.read_text('7.1.2\n')))
	yield (api.test(platform + '_binutils_test_fail') +
	api.platform.name(platform) +
	api.gitiles.refs('binutils refs',
	(BINUTILS_REF, binutils_revision)) +
	api.gitiles.refs('gcc refs',
	(GCC_REF, gcc_revision)) +
	api.step_data('cipd search fuchsia/gcc/${platform} ' +
	'git_revision:' + cipd_revision,
	api.json.output({'result': []})) +
	api.step_data('check for changes other than bfd/version.h',
	api.raw_io.stream_output('bfd/version.h\nothers\n',
	name='changed files')) +
	api.step_data('check for changes other than gcc/DATESTAMP',
	api.raw_io.stream_output('gcc/DATESTAMP\nothers\n',
	name='changed files')) +
	api.step_data('test x86_64 binutils', retcode=1))
	yield (api.test(platform + '_gcc_test_fail') +
	api.platform.name(platform) +
	api.gitiles.refs('binutils refs',
	(BINUTILS_REF, binutils_revision)) +
	api.gitiles.refs('gcc refs',
	(GCC_REF, gcc_revision)) +
	api.step_data('cipd search fuchsia/gcc/${platform} ' +
	'git_revision:' + cipd_revision,
	api.json.output({'result': []})) +
	api.step_data('check for changes other than bfd/version.h',
	api.raw_io.stream_output('bfd/version.h\nothers\n',
	name='changed files')) +
	api.step_data('check for changes other than gcc/DATESTAMP',
	api.raw_io.stream_output('', retcode=1,
	name='changed files')) +
	api.step_data('test aarch64 gcc', retcode=1))
	yield (api.test(platform + '_useless') +
	api.platform.name(platform) +
	api.gitiles.refs('binutils refs',
	(BINUTILS_REF, binutils_revision)) +
	api.gitiles.refs('gcc refs',
	(GCC_REF, gcc_revision)) +
	api.step_data('cipd search fuchsia/gcc/${platform} ' +
	'git_revision:' + cipd_revision,
	api.json.output({'result': []})) +
	api.step_data('check for changes other than bfd/version.h',
	api.raw_io.stream_output('bfd/version.h\n',
	name='changed files')) +
	api.step_data('check for changes other than gcc/DATESTAMP',
	api.raw_io.stream_output('gcc/DATESTAMP\n',
	name='changed files')))