blob: 86615ee6e3e96eb080a62dc63bedfe6d99869f07 [file] [log] [blame]
# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building Clang toolchain."""
from recipe_engine.recipe_api import Property
from PB.go.chromium.org.luci.common.proto.srcman.manifest import Manifest
from google.protobuf import json_format
import re
DEPS = [
'fuchsia/git',
'fuchsia/gitiles',
'fuchsia/goma',
'fuchsia/gsutil',
'fuchsia/macos_sdk',
'fuchsia/upload',
'recipe_engine/buildbucket',
'recipe_engine/cipd',
'recipe_engine/context',
'recipe_engine/file',
'recipe_engine/isolated',
'recipe_engine/json',
'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
'recipe_engine/python',
'recipe_engine/runtime',
'recipe_engine/raw_io',
'recipe_engine/scheduler',
'recipe_engine/source_manifest',
'recipe_engine/step',
]
TARGET_TO_ARCH = {
'x64': 'x86_64',
'arm64': 'aarch64',
}
TARGETS = TARGET_TO_ARCH.keys()
PLATFORM_TO_TRIPLE = {
'linux-amd64': 'x86_64-linux-gnu',
'linux-arm64': 'aarch64-linux-gnu',
'mac-amd64': 'x86_64-apple-darwin',
}
PLATFORMS = PLATFORM_TO_TRIPLE.keys()
# TODO(TC-476): remove this and switch to normalized triples.
TRIPLE_TO_NORMALIZED_TRIPLE = {
'x86_64-linux-gnu': 'x86_64-unknown-linux-gnu',
'aarch64-linux-gnu': 'aarch64-unknown-linux-gnu',
'x86_64-apple-darwin': 'x86_64-apple-darwin',
}
LIBXML2_GIT = 'https://fuchsia.googlesource.com/third_party/libxml2'
ZLIB_GIT = 'https://fuchsia.googlesource.com/third_party/zlib'
CIPD_SERVER_HOST = 'chrome-infra-packages.appspot.com'
PROPERTIES = {
'repository':
Property(
kind=str,
help='Git repository URL',
default='https://fuchsia.googlesource.com/third_party/llvm-project'
),
'revision':
Property(kind=str, help='Git revision', default=None),
'platform':
Property(kind=str, help='CIPD platform for the target', default=None),
}
def build_zlib(api, destdir, manifest, env=None):
zlib_dir = api.path['start_dir'].join('zlib')
src_dir = zlib_dir.join('src')
revision = api.git.checkout(ZLIB_GIT, src_dir, ref='refs/tags/v1.2.9')
git_checkout = manifest.directories[str(src_dir)].git_checkout
git_checkout.repo_url = ZLIB_GIT
git_checkout.revision = revision
git_checkout.fetch_ref = 'refs/tags/v1.2.9'
obj_dir = zlib_dir.join('obj')
api.file.ensure_directory('make objdir', obj_dir)
with api.context(cwd=obj_dir, env=env):
api.step('configure', [
src_dir.join('configure'),
'--prefix=',
'--static',
])
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install', 'DESTDIR=%s' % destdir])
def build_libxml2(api, host, target, destdir, manifest, env=None):
libxml2_dir = api.path['start_dir'].join('libxml2')
src_dir = libxml2_dir.join('src')
revision = api.git.checkout(LIBXML2_GIT, src_dir, ref='refs/tags/v2.9.8')
git_checkout = manifest.directories[str(src_dir)].git_checkout
git_checkout.repo_url = LIBXML2_GIT
git_checkout.revision = revision
git_checkout.fetch_ref = 'refs/tags/v2.9.8'
with api.context(cwd=src_dir):
api.step('autoconf', ['autoreconf', '-i', '-f'])
obj_dir = libxml2_dir.join('obj')
api.file.ensure_directory('make objdir', obj_dir)
with api.context(cwd=obj_dir):
api.step('configure', [
src_dir.join('configure'),
'--prefix=',
'--build=%s' % host,
'--host=%s' % target,
'--disable-shared',
'--enable-static',
'--with-zlib=%s' % destdir,
'--without-icu',
'--without-lzma',
'--without-python',
] + ['%s=%s' % (k, v) for k, v in env.iteritems()])
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install', 'DESTDIR=%s' % destdir])
def RunSteps(api, repository, revision, platform):
api.goma.ensure()
prod = api.buildbucket.builder_id.bucket == 'prod'
ci = api.buildbucket.builder_id.bucket == 'ci'
gitiles_commit = api.buildbucket.build_input.gitiles_commit
if gitiles_commit.host and gitiles_commit.project and gitiles_commit.id:
repository = 'https://%s/%s' % (gitiles_commit.host, gitiles_commit.project)
revision = gitiles_commit.id
# TODO: factor this out into a host_build recipe module.
host_platform = '%s-%s' % (api.platform.name.replace('win', 'windows'), {
'intel': {
32: '386',
64: 'amd64',
},
'arm': {
32: 'armv6',
64: 'arm64',
},
}[api.platform.arch][api.platform.bits])
target_platform = platform or host_platform
manifest = Manifest()
with api.step.nest('ensure_packages'):
with api.context(infra_steps=True):
cipd_dir = api.path['start_dir'].join('cipd')
pkgs = api.cipd.EnsureFile()
pkgs.add_package('infra/cmake/${platform}', 'version:3.9.2')
pkgs.add_package('infra/ninja/${platform}', 'version:1.8.2')
pkgs.add_package('fuchsia/third_party/clang/${platform}', 'goma')
pkgs.add_package('fuchsia/sysroot/linux-386', 'latest', 'linux-i386')
pkgs.add_package('fuchsia/sysroot/linux-amd64', 'latest', 'linux-amd64')
pkgs.add_package('fuchsia/sysroot/linux-arm64', 'latest', 'linux-arm64')
pkgs.add_package('fuchsia/sysroot/linux-armv6l', 'latest', 'linux-armhf')
pkgs.add_package('fuchsia/sdk/core/${platform}', 'latest', 'sdk')
ensured = api.cipd.ensure(cipd_dir, pkgs)
for subdir, pins in ensured.iteritems():
directory = manifest.directories[str(cipd_dir.join(subdir))]
directory.cipd_server_host = CIPD_SERVER_HOST
for pin in pins:
directory.cipd_package[pin.package].instance_id = pin.instance_id
staging_dir = api.path.mkdtemp('clang')
pkg_name = 'clang-%s' % api.platform.name.replace('mac', 'darwin')
pkg_dir = staging_dir.join(pkg_name)
api.file.ensure_directory('create pkg dir', pkg_dir)
with api.context(infra_steps=True):
llvm_dir = api.path['start_dir'].join('llvm-project')
revision = api.git.checkout(
repository,
path=llvm_dir,
ref=revision,
step_test_data=lambda: api.raw_io.test_api.stream_output(revision))
git_checkout = manifest.directories[str(llvm_dir)].git_checkout
git_checkout.repo_url = repository
git_checkout.revision = revision
lib_install_dir = staging_dir.join('lib_install')
api.file.ensure_directory('create lib_install_dir', lib_install_dir)
target_triple = PLATFORM_TO_TRIPLE[target_platform]
host_triple = PLATFORM_TO_TRIPLE[host_platform]
with api.macos_sdk(), api.goma.build_with_goma():
if api.platform.name == 'linux':
host_sysroot = cipd_dir.join(host_platform)
target_sysroot = cipd_dir.join(target_platform)
elif api.platform.name == 'mac':
# TODO(IN-148): Eventually use our own hermetic sysroot as for Linux.
step_result = api.step(
'xcrun', ['xcrun', '--show-sdk-path'],
stdout=api.raw_io.output(name='sdk-path', add_output_log=True),
step_test_data=lambda: api.raw_io.test_api.stream_output(
'/some/xcode/path'))
target_sysroot = host_sysroot = step_result.stdout.strip()
else: # pragma: no cover
assert False, 'unsupported platform'
env = {
'CC':
'%s %s' %
(api.goma.goma_dir.join('gomacc'), cipd_dir.join('bin', 'clang')),
'CFLAGS':
'-O3 -fPIC --target=%s --sysroot=%s' %
(target_triple, target_sysroot),
'AR':
cipd_dir.join('bin', 'llvm-ar'),
'NM':
cipd_dir.join('bin', 'llvm-nm'),
'RANLIB':
cipd_dir.join('bin', 'llvm-ranlib'),
}
with api.step.nest('zlib'):
build_zlib(api, lib_install_dir, manifest, env)
with api.step.nest('libxml2'):
build_libxml2(api, host_triple, target_triple, lib_install_dir, manifest,
env)
json_manifest = json_format.MessageToJson(
manifest, preserving_proto_field_name=True)
api.file.write_text('source manifest', pkg_dir.join('source_manifest.json'),
json_manifest)
api.source_manifest.set_json_manifest('checkout', json_manifest)
# build clang+llvm
build_dir = staging_dir.join('llvm_build_dir')
api.file.ensure_directory('create llvm build dir', build_dir)
arguments = {
'cc': cipd_dir.join('bin', 'clang'),
'cxx': cipd_dir.join('bin', 'clang++'),
'ar': cipd_dir.join('bin', 'llvm-ar'),
'ld': cipd_dir.join('bin', 'ld.lld'),
'nm': cipd_dir.join('bin', 'llvm-nm'),
'objcopy': cipd_dir.join('bin', 'llvm-objcopy'),
'objdump': cipd_dir.join('bin', 'llvm-objdump'),
'ranlib': cipd_dir.join('bin', 'llvm-ranlib'),
'strip': cipd_dir.join('bin', 'llvm-strip'),
'gomacc': api.goma.goma_dir.join('gomacc'),
'ninja': cipd_dir.join('ninja'),
'target_triple': TRIPLE_TO_NORMALIZED_TRIPLE[target_triple],
'host_triple': host_triple,
'target_sysroot': target_sysroot,
'host_sysroot': host_sysroot,
'pkgroot_include_dir': lib_install_dir.join('include'),
'libxml2_include_dir': lib_install_dir.join('include', 'libxml2'),
'platform_lib_dir': cipd_dir.join(target_platform, 'lib'),
'pkgroot_lib_dir': lib_install_dir.join('lib'),
'linux_arm64_sysroot': cipd_dir.join('linux-arm64'),
'linux_armhf_sysroot': cipd_dir.join('linux-armhf'),
'linux_i386_sysroot': cipd_dir.join('linux-i386'),
'linux_amd64_sysroot': cipd_dir.join('linux-amd64'),
'fuchsia_sdk': cipd_dir.join('sdk'),
}
arguments.update({
'BOOTSTRAP_': 'BOOTSTRAP_',
'STAGE2_': 'STAGE2_',
} if prod else {
'BOOTSTRAP_': '',
'STAGE2_': '',
})
llvm_projects = ['clang', 'clang-tools-extra', 'lld']
llvm_runtimes = ['compiler-rt', 'libcxx', 'libcxxabi', 'libunwind']
options = [
'-GNinja',
'-DCMAKE_C_COMPILER_LAUNCHER={gomacc}',
'-DCMAKE_CXX_COMPILER_LAUNCHER={gomacc}',
'-DCMAKE_ASM_COMPILER_LAUNCHER={gomacc}',
'-DCMAKE_C_COMPILER={cc}',
'-DCMAKE_CXX_COMPILER={cxx}',
'-DCMAKE_ASM_COMPILER={cc}',
'-DCMAKE_MAKE_PROGRAM={ninja}',
'-DCMAKE_INSTALL_PREFIX=',
'-DLLVM_ENABLE_PROJECTS=%s' % ';'.join(llvm_projects),
'-DLLVM_ENABLE_RUNTIMES=%s' % ';'.join(llvm_runtimes),
]
options.extend({
'linux': [
'-DCMAKE_AR={ar}',
'-DCMAKE_LINKER={ld}',
'-DCMAKE_NM={nm}',
'-DCMAKE_OBJCOPY={objcopy}',
'-DCMAKE_OBJDUMP={objdump}',
'-DCMAKE_RANLIB={ranlib}',
'-DCMAKE_STRIP={strip}',
],
'mac': [],
}[api.platform.name])
if api.platform.name == 'linux':
options.extend([
# BOOTSTRAP_ prefixed flags are passed to the second stage compiler.
'-D{BOOTSTRAP_}CMAKE_C_FLAGS=-I{pkgroot_include_dir} -I{libxml2_include_dir}',
'-D{BOOTSTRAP_}CMAKE_CXX_FLAGS=-I{pkgroot_include_dir} -I{libxml2_include_dir}',
'-D{BOOTSTRAP_}CMAKE_SHARED_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread -L{platform_lib_dir} -L{pkgroot_lib_dir}',
'-D{BOOTSTRAP_}CMAKE_MODULE_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread -L{platform_lib_dir} -L{pkgroot_lib_dir}',
'-D{BOOTSTRAP_}CMAKE_EXE_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread -L{platform_lib_dir} -L{pkgroot_lib_dir}',
'-D{BOOTSTRAP_}CMAKE_SYSROOT={target_sysroot}',
])
if prod:
options.extend([
# Unprefixed flags are used by the first stage compiler.
'-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread',
'-DCMAKE_MODULE_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread',
'-DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ -ldl -lpthread',
'-DCMAKE_SYSROOT={host_sysroot}',
])
else:
options.extend([
'-DLLVM_ENABLE_LTO=OFF',
])
if host_triple != target_triple:
options.extend([
'-D{BOOTSTRAP_}CMAKE_SYSTEM_NAME=Linux',
'-D{BOOTSTRAP_}CMAKE_C_COMPILER_TARGET={target_triple}',
'-D{BOOTSTRAP_}CMAKE_CXX_COMPILER_TARGET={target_triple}',
'-D{BOOTSTRAP_}LLVM_DEFAULT_TARGET_TRIPLE={target_triple}',
])
elif api.platform.name == 'mac':
options.extend([
'-D{BOOTSTRAP_}CMAKE_%s_LINKER_FLAGS=-nostdlib++ %s' %
(mode, cipd_dir.join('lib', 'libc++.a'))
for mode in ['SHARED', 'MODULE', 'EXE']
])
if prod:
options.extend([
'-DCMAKE_%s_LINKER_FLAGS=-nostdlib++ %s' %
(mode, cipd_dir.join('lib', 'libc++.a'))
for mode in ['SHARED', 'MODULE', 'EXE']
])
else:
options.extend([
'-DLLVM_ENABLE_LTO=OFF',
])
# STAGE2_ prefixed flags are passed to the second stage by the first stage build.
if ci:
options.extend([
'-D{STAGE2_}LLVM_ENABLE_ASSERTIONS=ON',
])
options.extend([
'-D{STAGE2_}LINUX_aarch64-unknown-linux-gnu_SYSROOT={linux_arm64_sysroot}',
'-D{STAGE2_}LINUX_armv7-unknown-linux-gnueabihf_SYSROOT={linux_armhf_sysroot}',
'-D{STAGE2_}LINUX_i386-unknown-linux-gnu_SYSROOT={linux_i386_sysroot}',
'-D{STAGE2_}LINUX_x86_64-unknown-linux-gnu_SYSROOT={linux_amd64_sysroot}',
'-D{STAGE2_}FUCHSIA_SDK={fuchsia_sdk}',
])
with api.step.nest('clang'), api.context(cwd=build_dir):
api.step(
'configure', [cipd_dir.join('bin', 'cmake')] +
[option.format(**arguments) for option in options] + [
'-C',
llvm_dir.join('clang', 'cmake', 'caches', 'Fuchsia%s.cmake' %
('' if prod else '-stage2')),
llvm_dir.join('llvm')
])
# Build and run tests.
# TODO: we should be running check-all
projects = ['llvm', 'clang', 'lld']
api.step('build', [cipd_dir.join('ninja'), '-j%d' % api.goma.jobs])
api.step('test', [cipd_dir.join('ninja'),
'-j%d' % api.goma.jobs] +
['check-%s' % project for project in projects])
# Build the full (two-stage) distribution.
target_prefix = 'stage2-' if prod else ''
api.step(
'distribution',
[
cipd_dir.join('ninja'),
# This only applies to the first stage, second stage is invoked by
# CMake as a subprocess and will use Ninja's default.
'-j%d' % api.goma.jobs,
target_prefix + 'distribution',
])
with api.context(env={'DESTDIR': pkg_dir}):
api.step('install', [
cipd_dir.join('ninja'),
target_prefix + 'install-distribution-stripped',
])
step_result = api.file.read_text(
'Version.inc',
build_dir.join(*(
(['tools', 'clang', 'stage2-bins'] if prod else []) +
['tools', 'clang', 'include', 'clang', 'Basic', 'Version.inc'])),
test_data='#define CLANG_VERSION_STRING "8.0.0"')
m = re.search(r'CLANG_VERSION_STRING "([a-zA-Z0-9.-]+)"', step_result)
assert m, 'Cannot determine Clang version'
clang_version = m.group(1)
# TODO(TC-366): Ideally this would be done by the cmake build itself.
# //zircon/public/gn/toolchain/clang.gni:clang_runtime sets the JSON schema.
resource_dir_switch = '--resource-dir=clang/%s' % clang_version
runtime_targets = [[
'--target',
'%s-unknown-fuchsia' % arch,
'--alias',
'%s-fuchsia' % arch,
] for arch in TARGET_TO_ARCH.itervalues()]
api.python(
'generate runtimes.json',
script=api.resource('runtimes.py'),
args=[
'--dir', pkg_dir, '--readelf',
cipd_dir.join('bin', 'llvm-readelf'), resource_dir_switch
] + [t for ts in runtime_targets for t in ts],
stdout=api.json.output(
leak_to=pkg_dir.join('lib', 'runtime.json'), name='runtime.json'))
for target in runtime_targets:
api.python(
'generate %s.manifest' % target[-1],
script=api.resource('runtimes.py'),
args=[
'--dir', pkg_dir, '--readelf',
cipd_dir.join('bin', 'llvm-readelf'), resource_dir_switch,
'--manifest'
] + target,
stdout=api.raw_io.output(
leak_to=pkg_dir.join('lib', '%s.manifest' % target[-1])))
api.python(
'generate license',
api.resource('generate_license.py'),
args=['--include'] + [
llvm_dir.join(project, 'LICENSE.TXT')
for project in llvm_projects + ['llvm'] + llvm_runtimes
] + [
'--extract',
api.path['start_dir'].join('libxml2', 'src', 'Copyright'),
'-',
'--extract',
api.path['start_dir'].join('zlib', 'src', 'zlib.h'),
'4-22',
],
stdout=api.raw_io.output(leak_to=pkg_dir.join('LICENSE')))
isolated = api.upload.upload_isolated(pkg_dir)
if prod:
# The published package has the same name for every platform.
pkg_name = 'clang'
api.upload.cipd_package(
'fuchsia/third_party/%s/%s' % (pkg_name, target_platform),
pkg_dir, [api.upload.DirectoryPath(pkg_dir)],
{'git_revision': revision},
repository=repository,
extra_tags={'version': clang_version})
host = 'fuchsia.googlesource.com'
project = 'fuchsia'
refs = api.gitiles.refs('https://%s/%s' % (host, project))
ref = 'refs/heads/master'
revision = refs.get(ref, 'HEAD')
# Do a full integration build. This will use the just-built toolchain
# to build all of Fuchsia to check whether there are any regressions.
api.scheduler.emit_trigger(
api.scheduler.BuildbucketTrigger(
properties={
'clang_toolchain': {
'git_repository': repository,
'git_revision': revision,
},
'build.clang_toolchain': {
'type': 'isolated',
'instance': isolated,
},
},
tags={
'buildset':
'commit/gitiles/%s/%s/+/%s' % (host, project, revision),
'gitiles_ref':
ref,
}),
project='fuchsia',
jobs={
'linux-amd64': (
'fuchsia-arm64-debug-clang',
'fuchsia-arm64-release-clang',
'fuchsia-x64-debug-clang',
'fuchsia-x64-release-clang',
),
'mac-amd64': ('fuchsia-host-mac-clang',),
}[target_platform])
def GenTests(api):
for os in ('linux', 'mac'):
yield (api.test('ci-%s-x64' % os) + api.buildbucket.ci_build(
project='fuchsia',
bucket='ci',
git_repo='https://fuchsia.googlesource.com/third_party/llvm-project',
revision='a' * 40,
) + api.runtime(is_luci=True, is_experimental=False) +
api.platform.name(os) + api.properties(platform=os + '-amd64'))
yield (api.test('prod-%s-x64' % os) + api.buildbucket.ci_build(
project='fuchsia',
bucket='prod',
git_repo='https://fuchsia.googlesource.com/third_party/llvm-project',
revision='a' * 40,
) + api.runtime(is_luci=True, is_experimental=False) +
api.platform.name(os) + api.properties(platform=os + '-amd64') +
api.gitiles.refs('refs', ('refs/heads/master', 'b' * 40)))
yield (api.test('linux-arm64') + api.buildbucket.ci_build(
project='fuchsia',
bucket='ci',
git_repo='https://fuchsia.googlesource.com/third_party/llvm-project',
revision='a' * 40,
) + api.runtime(is_luci=True, is_experimental=False) +
api.platform.name('linux') + api.properties(platform='linux-arm64'))