| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building GCC toolchain.""" |
| |
| from recipe_engine.recipe_api import StepFailure |
| |
| import re |
| |
| DEPS = [ |
| 'fuchsia/git', |
| 'fuchsia/goma', |
| 'fuchsia/hash', |
| 'fuchsia/jiri', |
| 'fuchsia/macos_sdk', |
| 'fuchsia/upload', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/platform', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| 'recipe_engine/url', |
| ] |
| |
| BINUTILS_GIT = 'https://gnu.googlesource.com/binutils-gdb' |
| BINUTILS_REF = 'refs/tags/binutils-2_33_1' |
| |
| GCC_GIT = 'https://gnu.googlesource.com/gcc' |
| GCC_REF = 'refs/tags/gcc-9_1_0-release' |
| |
| # Bump this number whenever changing this recipe in ways that affect the |
| # package built without also changing any upstream revision pin. |
| # TODO(crbug.com/947158): Remove this when the recipes repo/rev are available. |
| RECIPE_SALT = '1' |
| |
| |
| def RunSteps(api): |
| api.goma.ensure() |
| |
| binutils_dir = api.path['start_dir'].join('binutils-gdb') |
| gcc_dir = api.path['start_dir'].join('gcc') |
| |
| projects = { |
| 'binutils-gdb': (BINUTILS_GIT, binutils_dir, BINUTILS_REF), |
| 'gcc': (GCC_GIT, gcc_dir, GCC_REF), |
| } |
| |
| gitiles_commit = api.buildbucket.build_input.gitiles_commit |
| if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id: |
| projects[gitiles_commit.project] = ( |
| 'https://%s/%s' % (gitiles_commit.host, gitiles_commit.project), |
| api.path['start_dir'].join(gitiles_commit.project), gitiles_commit.id) |
| |
| with api.context(infra_steps=True): |
| with api.step.nest('binutils-gdb'): |
| binutils_revision = api.git.checkout(*projects['binutils-gdb']) |
| with api.step.nest('gcc'): |
| gcc_revision = api.git.checkout(*projects['gcc']) |
| |
| with api.step.nest('ensure_packages'): |
| with api.context(infra_steps=True): |
| pkgs = api.cipd.EnsureFile() |
| pkgs.add_package('fuchsia/third_party/clang/${platform}', 'goma') |
| if api.platform.name == 'linux': |
| pkgs.add_package('fuchsia/sysroot/${platform}', 'latest') |
| cipd_dir = api.path['start_dir'].join('cipd') |
| api.cipd.ensure(cipd_dir, pkgs) |
| |
| if api.platform.name == 'linux': |
| host_sysroot = cipd_dir |
| elif api.platform.name == 'mac': |
| # TODO(IN-148): Eventually use our own hermetic sysroot as for Linux. |
| step_result = api.step( |
| 'xcrun', ['xcrun', '--show-sdk-path'], |
| stdout=api.raw_io.output(name='sdk-path', add_output_log=True), |
| step_test_data=lambda: api.raw_io.test_api.stream_output( |
| '/some/xcode/path')) |
| host_sysroot = step_result.stdout.strip() |
| else: # pragma: no cover |
| assert False, 'what platform?' |
| |
| with api.context(cwd=gcc_dir): |
| # download GCC dependencies: GMP, ISL, MPC and MPFR libraries |
| api.step('download prerequisites', |
| [gcc_dir.join('contrib', 'download_prerequisites')]) |
| |
| staging_dir = api.path.mkdtemp('gcc') |
| pkg_name = 'gcc-%s' % api.platform.name.replace('mac', 'darwin') |
| pkg_dir = staging_dir.join(pkg_name) |
| api.file.ensure_directory('create pkg dir', pkg_dir) |
| |
| # Some of the makefile logic splits $CC at its first word and injects |
| # a switch there. So make $CC and $CXX be single words by writing |
| # little scripts. Autoconf does some checks with CPPFLAGS but not |
| # CFLAGS and other checks with CFLAGS but not CPPFLAGS. The sysroot |
| # is necessary for all cases, so fold that into the script too so it's |
| # impossible to omit it in any $CC or $CXX invocation. |
| cc_path = staging_dir.join('host-cc') |
| cxx_path = staging_dir.join('host-cxx') |
| for script, compiler in [(cc_path, 'clang'), (cxx_path, 'clang++')]: |
| api.file.write_text( |
| 'write %s script' % api.path.basename(script), script, """#!/bin/sh |
| exec %s %s --sysroot=%s "$@" |
| """ % (api.goma.goma_dir.join('gomacc'), cipd_dir.join('bin', |
| compiler), host_sysroot)) |
| api.step('make %s executable' % api.path.basename(script), |
| ['chmod', '+x', script]) |
| |
| host_cflags = '-O3' |
| if api.platform.name != 'mac': |
| # LTO works for binutils on Linux but fails on macOS. |
| host_cflags += ' -flto' |
| host_compiler_args = { |
| 'CC': '%s' % cc_path, |
| 'CXX': '%s' % cxx_path, |
| 'CFLAGS': host_cflags, |
| 'CXXFLAGS': host_cflags, |
| } |
| |
| if api.platform.name == 'mac': |
| # Our host toolchain for Mac provides static libc++ but doesn't |
| # know how to link it in by itself. |
| host_compiler_args['CXXFLAGS'] += ' -nostdlib++' |
| host_compiler_args['LIBS'] = cipd_dir.join('lib', 'libc++.a') |
| else: |
| # Always link libc++ statically in case a shared library is available. |
| host_compiler_args['CXXFLAGS'] += ' -static-libstdc++' |
| |
| host_compiler_args = sorted( |
| '%s=%s' % item for item in host_compiler_args.iteritems()) |
| |
| # We force LIMITS_H_TEST=true to tell the compiler to install a <limits.h> |
| # that requires a system <limits.h> it can find via #include_next. But we |
| # don't have any system headers for the just-built compiler building target |
| # code (i.e. libgcc). So mock up a little include directory that contains |
| # just an empty <limits.h> for it to find. |
| mock_include_dir = staging_dir.join('mock-include') |
| api.file.ensure_directory('create mock include dir', mock_include_dir) |
| api.file.write_text('write dummy <limits.h>', |
| mock_include_dir.join('limits.h'), '/* dummy */') |
| |
| with api.macos_sdk(), api.goma.build_with_goma(): |
| for target, enable_targets in [('aarch64', 'arm-eabi'), |
| ('x86_64', 'x86_64-pep')]: |
| # configure arguments that are the same for binutils and gcc. |
| common_args = host_compiler_args + [ |
| '--prefix=', # we're building a relocatable package |
| '--target=%s-elf' % target, |
| '--enable-initfini-array', # Fuchsia uses .init/.fini arrays |
| '--enable-gold', # Zircon uses gold for userspace build |
| # Enable plugins and threading for Gold. |
| # This also happens to make it explicitly link in -lpthread and -dl, |
| # which are required by host_clang's static libc++. |
| '--enable-plugins', |
| '--enable-threads', |
| '--disable-werror', # ignore warnings reported by Clang |
| '--disable-nls', # no need for localization |
| '--with-included-gettext', # use include gettext library |
| ] |
| |
| # build binutils |
| binutils_build_dir = staging_dir.join('binutils_%s_build_dir' % target) |
| api.file.ensure_directory('create binutils %s build dir' % target, |
| binutils_build_dir) |
| |
| with api.context(cwd=binutils_build_dir): |
| |
| def binutils_make_step(name, make_target, prefix, jobs, make_args=None): |
| # As of 2.32, gold/testsuite/Makefile.am unconditionally edits in |
| # a -B.../ switch to make the compiler use the just-built gold as |
| # the linker for test suite binaries. This is wrong when |
| # building a cross-linker. Force it to a no-op on the make |
| # command line to work around the bug. |
| # TODO(mcgrathr): Drop this when we roll to a binutils that has |
| # this fixed upstream. |
| make_args = make_args or [] |
| make_args.append('MAKEOVERRIDES=editcc=-eb') |
| return api.step( |
| '%s %s binutils' % (name, make_target), |
| ['make', '-j%s' % jobs] + make_args + [ |
| '%s-%s' % (prefix, component) |
| for component in ['binutils', 'gas', 'ld', 'gold'] |
| ]) |
| |
| api.step( |
| 'configure %s binutils' % target, |
| [ |
| binutils_dir.join('configure'), |
| '--enable-deterministic-archives', # more deterministic builds |
| '--enable-targets=%s' % enable_targets, |
| ] + common_args) |
| binutils_make_step('build', target, 'all', api.goma.jobs) |
| try: |
| binutils_make_step('test', target, 'check', api.platform.cpu_count, |
| ['-k']) |
| except StepFailure as error: |
| logs = { |
| l[0]: |
| api.file.read_text('binutils %s %s' % (target, '/'.join(l)), |
| binutils_build_dir.join(*l)).splitlines() |
| for l in [ |
| ('gas', 'testsuite', 'gas.log'), |
| ('binutils', 'binutils.log'), |
| ('ld', 'ld.log'), |
| ('gold', 'testsuite', 'test-suite.log'), |
| ] |
| } |
| step_result = api.step('binutils test failure', cmd=None) |
| for name, text in logs.iteritems(): |
| step_result.presentation.logs[name] = text |
| raise error |
| binutils_make_step('install', target, 'install-strip', 1, |
| ['DESTDIR=%s' % pkg_dir]) |
| |
| # build gcc |
| gcc_build_dir = staging_dir.join('gcc_%s_build_dir' % target) |
| api.file.ensure_directory('create gcc %s build dir' % target, |
| gcc_build_dir) |
| |
| with api.context( |
| cwd=gcc_build_dir, env_prefixes={'PATH': [pkg_dir.join('bin')]}): |
| gcc_goals = ['gcc', 'target-libgcc'] |
| |
| def gcc_make_step(name, make_target, jobs, args): |
| cmd = [ |
| 'make', |
| '-j%s' % jobs, |
| 'MAKEOVERRIDES=USE_GCC_STDINT=provide LIMITS_H_TEST=true', |
| # Force flags for libgcc to get <limits.h> kludge. |
| 'CPPFLAGS_FOR_TARGET=-idirafter %s' % mock_include_dir, |
| ] |
| return api.step('%s %s gcc' % (name, make_target), cmd + args) |
| |
| api.step( |
| 'configure %s gcc' % target, |
| [ |
| gcc_dir.join('configure'), |
| '--enable-languages=c,c++', |
| '--disable-libstdcxx', # we don't need libstdc++ |
| '--disable-libssp', # we don't need libssp either |
| '--disable-libquadmath', # and nor do we need libquadmath |
| ] + common_args) |
| gcc_make_step('build', target, api.goma.jobs, |
| ['all-%s' % goal for goal in gcc_goals]) |
| try: |
| gcc_make_step('test', target, api.platform.cpu_count, ['check-gcc']) |
| finally: |
| logs = { |
| l[-1]: api.file.read_text('gcc %s %s' % (target, '/'.join(l)), |
| gcc_build_dir.join(*l)).splitlines() |
| for l in [ |
| ('gcc', 'testsuite', 'gcc', 'gcc.log'), |
| ('gcc', 'testsuite', 'g++', 'g++.log'), |
| ] |
| } |
| step_result = api.step('test %s gcc logs' % target, cmd=None) |
| for name, text in logs.iteritems(): |
| step_result.presentation.logs[name] = text |
| gcc_make_step('install', target, 1, ['DESTDIR=%s' % pkg_dir] + |
| ['install-strip-%s' % goal for goal in gcc_goals]) |
| |
| binutils_version = api.file.read_text( |
| 'binutils version', |
| binutils_dir.join('bfd', 'version.m4'), |
| test_data='m4_define([BFD_VERSION], [2.27.0])') |
| m = re.match(r'm4_define\(\[BFD_VERSION\], \[([^]]+)\]\)', binutils_version) |
| assert m and m.group(1), ('bfd/version.m4 has unexpected format: %r' % |
| binutils_version) |
| binutils_version = m.group(1) |
| gcc_version = api.file.read_text('gcc version', |
| gcc_dir.join('gcc', 'BASE-VER')).rstrip() |
| |
| cipd_git_repository = ','.join([GCC_GIT, BINUTILS_GIT]) |
| cipd_git_revision = ','.join([gcc_revision, binutils_revision]) |
| |
| # Add a bogus "salt" repository/revision to represent recipe changes when |
| # the upstream revisions haven't changed. Bump this number whenever |
| # changing this recipe in ways that affect the package built without also |
| # changing any upstream revision pin. |
| # TODO(crbug.com/947158): Replace this with the recipes repo/rev when |
| # infra makes that available here. |
| if RECIPE_SALT: # pragma: no cover |
| cipd_git_repository += ',<salt>' |
| cipd_git_revision += ',%s' % RECIPE_SALT |
| |
| version = ','.join([gcc_version, binutils_version]) |
| api.upload.cipd_package( |
| 'fuchsia/gcc/${platform}', |
| pkg_dir, [api.upload.DirectoryPath(pkg_dir)], |
| {'git_revision': cipd_git_revision}, |
| repository=cipd_git_repository, |
| extra_tags={'version': version}) |
| |
| |
| def GenTests(api): |
| binutils_revision = '3d861fdb826c2f5cf270dd5f585d0e6057e1bf4f' |
| gcc_revision = '4b5e15daff8b54440e3fda451c318ad31e532fab' |
| cipd_revision = ','.join([gcc_revision, binutils_revision]) |
| if RECIPE_SALT: # pragma: no cover |
| cipd_revision += ',%s' % RECIPE_SALT |
| |
| for platform in ('linux', 'mac'): |
| yield (api.test(platform) + api.platform.name(platform) + |
| api.step_data('binutils-gdb.git rev-parse', |
| api.raw_io.stream_output(binutils_revision)) + |
| api.step_data('gcc.git rev-parse', |
| api.raw_io.stream_output(gcc_revision))) |
| for salt in ['', 999] + ([RECIPE_SALT] if RECIPE_SALT else []): |
| salt = '%s' % salt |
| yield (api.test(platform + '_new' + salt) + api.platform.name(platform) + |
| api.step_data('binutils-gdb.git rev-parse', |
| api.raw_io.stream_output(binutils_revision)) + |
| api.step_data('gcc.git rev-parse', |
| api.raw_io.stream_output(gcc_revision)) + |
| api.step_data( |
| 'cipd.cipd search fuchsia/gcc/${platform} git_revision:' + |
| cipd_revision, api.json.output({'result': []})) + |
| api.step_data('gcc version', api.file.read_text('7.1.2\n'))) |
| yield (api.test(platform + '_binutils_test_fail' + salt) + |
| api.buildbucket.ci_build( |
| git_repo=BINUTILS_GIT, revision=binutils_revision) + |
| api.step_data('gcc.git rev-parse', |
| api.raw_io.stream_output(gcc_revision)) + |
| api.platform.name(platform) + |
| api.step_data('test x86_64 binutils', retcode=1)) |
| yield (api.test(platform + '_gcc_test_fail' + salt) + |
| api.buildbucket.ci_build(git_repo=GCC_GIT, revision=gcc_revision) + |
| api.step_data('binutils-gdb.git rev-parse', |
| api.raw_io.stream_output(binutils_revision)) + |
| api.platform.name(platform) + |
| api.step_data('test aarch64 gcc', retcode=1)) |
