blob: bbe182128a0b068d078e4e1a29573d01af734785 [file] [log] [blame]
# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building QEMU."""
from recipe_engine.post_process import StatusSuccess
from recipe_engine.recipe_api import Property
DEPS = [
'fuchsia/git',
'fuchsia/gitiles',
'fuchsia/goma',
'fuchsia/gsutil',
'fuchsia/upload',
'recipe_engine/cipd',
'recipe_engine/context',
'recipe_engine/isolated',
'recipe_engine/json',
'recipe_engine/file',
'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
'recipe_engine/raw_io',
'recipe_engine/step',
]
PLATFORM_TO_TRIPLE = {
'linux-amd64': 'x86_64-linux-gnu',
'linux-arm64': 'aarch64-linux-gnu',
'mac-amd64': 'x86_64-apple-darwin',
}
PLATFORMS = PLATFORM_TO_TRIPLE.keys()
PROPERTIES = {
'repository':
Property(
kind=str,
help='Git repository URL',
default='https://fuchsia.googlesource.com/third_party/qemu'),
'branch':
Property(kind=str, help='Git branch', default='refs/heads/master'),
'revision':
Property(kind=str, help='Revision', default=None),
'platform':
Property(kind=str, help='CIPD platform for the target', default=None),
'prod':
Property(
kind=bool, help='Whether to do production build', default=True),
}
def platform_sysroot(api, cipd_dir, platform):
if platform.startswith('linux'):
return cipd_dir.join(platform)
elif platform.startswith('mac'): # pragma: no cover
# TODO(IN-148): Eventually use our own hermetic sysroot as for Linux.
step_result = api.step(
'xcrun', ['xcrun', '--sdk', 'macosx', '--show-sdk-path'],
stdout=api.raw_io.output(name='sdk-path', add_output_log=True),
step_test_data=lambda: api.raw_io.test_api.stream_output(
'/some/xcode/path'))
return step_result.stdout.strip()
assert False, 'unsupported platform' # pragma: no cover
return None # pragma: no cover
def environment(api,
cipd_dir,
platform,
host,
cflags=(),
cxxflags=(),
ldflags=()):
sysroot = ['--sysroot=%s' % platform_sysroot(api, cipd_dir, platform)]
target = ['--target=%s' %
PLATFORM_TO_TRIPLE[platform]] if platform != host else []
variables = {
'CC': cipd_dir.join('bin', 'clang'),
'CXX': cipd_dir.join('bin', 'clang++'),
'CFLAGS': ' '.join(sysroot + target + list(cflags)),
'CPPFLAGS': ' '.join(sysroot + target + list(cflags)),
'CXXFLAGS': ' '.join(sysroot + target + list(cxxflags)),
'LDFLAGS': ' '.join(sysroot + target + list(ldflags)),
}
if platform.startswith('linux'):
variables.update({
'AR': cipd_dir.join('bin', 'llvm-ar'),
'RANLIB': cipd_dir.join('bin', 'llvm-ranlib'),
'NM': cipd_dir.join('bin', 'llvm-nm'),
'STRIP': cipd_dir.join('bin', 'llvm-strip'),
'OBJCOPY': cipd_dir.join('bin', 'llvm-objcopy'),
})
return variables
def configure(api,
cipd_dir,
src_dir,
platform,
host,
flags=(),
cflags=(),
cxxflags=(),
ldflags=(),
step_name='configure'):
target = PLATFORM_TO_TRIPLE[platform]
variables = environment(api, cipd_dir, platform, host, cflags, cxxflags,
ldflags)
if platform != host:
flags.extend([
'--build=%s' % PLATFORM_TO_TRIPLE[host],
'--host=%s' % target,
])
return api.step(step_name, [
src_dir.join('configure'),
] + flags + ['%s=%s' % (k, v) for k, v in variables.iteritems()])
def cmake(api, cipd_dir, src_dir, platform, options=None, step_name='cmake'):
options = options or []
if platform.startswith('linux'):
options.extend([
'-DCMAKE_LINKER=%s' % cipd_dir.join('bin', 'ld.lld'),
'-DCMAKE_NM=%s' % cipd_dir.join('bin', 'llvm-nm'),
'-DCMAKE_OBJCOPY=%s' % cipd_dir.join('bin', 'llvm-objcopy'),
'-DCMAKE_OBJDUMP=%s' % cipd_dir.join('bin', 'llvm-objdump'),
'-DCMAKE_RANLIB=%s' % cipd_dir.join('bin', 'llvm-ranlib'),
'-DCMAKE_STRIP=%s' % cipd_dir.join('bin', 'llvm-strip'),
])
target = PLATFORM_TO_TRIPLE[platform]
return api.step(step_name, [
cipd_dir.join('bin', 'cmake'),
'-GNinja',
'-DCMAKE_MAKE_PROGRAM=%s' % cipd_dir.join('ninja'),
'-DCMAKE_C_COMPILER=%s' % cipd_dir.join('bin', 'clang'),
'-DCMAKE_C_COMPILER_TARGET=%s' % target,
'-DCMAKE_CXX_COMPILER=%s' % cipd_dir.join('bin', 'clang++'),
'-DCMAKE_CXX_COMPILER_TARGET=%s' % target,
'-DCMAKE_SYSROOT=%s' % platform_sysroot(api, cipd_dir, platform),
] + options + [src_dir])
def automake(api, name, cipd_dir, pkg_dir, platform, host):
src_dir = cipd_dir.join('source', name)
build_dir = api.path['start_dir'].join(name)
api.file.ensure_directory(name, build_dir)
with api.context(cwd=build_dir):
configure(api, cipd_dir, src_dir, platform, host, [
'--prefix=%s' % pkg_dir,
])
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install'])
def build_make(api, cipd_dir, pkg_dir, platform, host):
automake(api, 'make', cipd_dir, pkg_dir, platform, host)
def build_m4(api, cipd_dir, pkg_dir, platform, host):
automake(api, 'm4', cipd_dir, pkg_dir, platform, host)
def build_autoconf(api, cipd_dir, pkg_dir, platform, host):
automake(api, 'autoconf', cipd_dir, pkg_dir, platform, host)
def build_automake(api, cipd_dir, pkg_dir, platform, host):
automake(api, 'automake', cipd_dir, pkg_dir, platform, host)
def build_libtool(api, cipd_dir, pkg_dir, platform, host):
automake(api, 'libtool', cipd_dir, pkg_dir, platform, host)
def build_pkg_config(api, cipd_dir, pkg_dir, platform, host):
src_dir = cipd_dir.join('source', 'pkg-config')
build_dir = api.path['start_dir'].join('pkg-config')
api.file.ensure_directory('pkg-config', build_dir)
ldflags = ()
if host.startswith('mac'):
ldflags = ('-framework CoreFoundation', '-framework CoreServices')
with api.context(cwd=build_dir):
configure(
api,
cipd_dir,
src_dir,
platform,
host, [
'--prefix=%s' % pkg_dir,
'--with-internal-glib',
'--disable-host-tool',
'--disable-debug',
],
ldflags=ldflags)
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install'])
def build_zlib(api, cipd_dir, pkg_dir, platform, host):
src_dir = cipd_dir.join('source', 'zlib')
build_dir = api.path['start_dir'].join('zlib')
api.file.ensure_directory('zlib', build_dir)
variables = environment(api, cipd_dir, platform, host, cflags=('-fPIC',))
with api.context(cwd=build_dir, env=variables):
api.step('configure', [
src_dir.join('configure'),
'--prefix=',
'--static',
])
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install', 'DESTDIR=%s' % pkg_dir])
def build_pixman(api, cipd_dir, pkg_dir, platform, host):
src_dir = cipd_dir.join('source', 'pixman')
build_dir = api.path['start_dir'].join('pixman')
api.file.ensure_directory('pixman', build_dir)
with api.context(cwd=build_dir):
configure(api, cipd_dir, src_dir, platform, host, [
'--disable-dependency-tracking',
'--disable-gtk',
'--disable-shared',
'--disable-silent-rules',
'--enable-static',
'--prefix=',
'--with-pic',
])
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install', 'DESTDIR=%s' % pkg_dir])
def build_sdl(api, cipd_dir, pkg_dir, platform, host): # pylint: disable=unused-argument
# host isn't used, but keep it for symmetry with the other build_ functions.
src_dir = cipd_dir.join('source', 'sdl')
build_dir = api.path['start_dir'].join('sdl')
api.file.ensure_directory('sdl', build_dir)
with api.context(cwd=build_dir):
cmake(api, cipd_dir, src_dir, platform, [
'-DCMAKE_INSTALL_PREFIX=',
'-DVIDEO_WAYLAND=OFF',
'-DSDL_SHARED=OFF',
'-DSDL_STATIC_PIC=ON',
'-DGCC_ATOMICS=ON',
])
api.step('build', [cipd_dir.join('ninja')])
with api.context(env={'DESTDIR': pkg_dir}):
api.step('install', [cipd_dir.join('ninja'), 'install'])
def build_libffi(api, cipd_dir, pkg_dir, platform, host):
src_dir = cipd_dir.join('source', 'libffi')
build_dir = api.path['start_dir'].join('libffi')
api.file.ensure_directory('libffi', build_dir)
with api.context(cwd=build_dir):
configure(api, cipd_dir, src_dir, platform, host, [
'--disable-debug',
'--disable-dependency-tracking',
'--disable-shared',
'--enable-static',
'--prefix=',
'--target=%s' % PLATFORM_TO_TRIPLE[platform],
'--with-pic',
])
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install', 'DESTDIR=%s' % pkg_dir])
def build_ncurses(api, cipd_dir, pkg_dir, platform, host):
src_dir = cipd_dir.join('source', 'ncurses')
build_dir = api.path['start_dir'].join('ncurses')
api.file.ensure_directory('ncurses', build_dir)
with api.context(cwd=build_dir):
configure(
api,
cipd_dir,
src_dir,
platform,
host, [
'--disable-debug',
'--disable-dependency-tracking',
'--disable-shared',
'--enable-pc-files',
'--enable-sigwinch',
'--enable-widec',
'--without-gpm',
'--without-progs',
'--with-pic',
],
ldflags=('-ldl', '-lpthread'))
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install', 'DESTDIR=%s' % pkg_dir])
def build_gettext(api, cipd_dir, pkg_dir, platform, host):
src_dir = cipd_dir.join('source', 'gettext')
build_dir = api.path['start_dir'].join('gettext')
api.file.ensure_directory('gettext', build_dir)
with api.context(cwd=build_dir):
configure(api, cipd_dir, src_dir, platform, host, [
'--disable-dependency-tracking',
'--disable-silent-rules',
'--disable-debug',
'--disable-shared',
'--disable-java',
'--disable-csharp',
'--disable-c++',
'--disable-openmp',
'--enable-static',
'--prefix=',
'--with-pic',
'--with-included-gettext',
'--with-included-glib',
'--with-included-libcroco',
'--with-included-libunistring',
'--without-git',
'--without-cvs',
'--without-xz',
])
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install', 'DESTDIR=%s' % pkg_dir])
def build_glib(api, cipd_dir, pkg_dir, platform, host):
src_dir = cipd_dir.join('source', 'glib')
build_dir = api.path['start_dir'].join('glib')
api.file.ensure_directory('glib', build_dir)
extra_args = []
if platform != host:
api.file.write_text(
'cache', build_dir.join('%s.cache' % platform), """
glib_cv_long_long_format=ll
glib_cv_stack_grows=no
glib_cv_uscore=no
""")
extra_args.append('--cache-file=%s.cache' % platform)
with api.context(cwd=src_dir, env={'NOCONFIGURE': '1'}):
api.step('autogen', [src_dir.join('autogen.sh')])
with api.context(cwd=build_dir):
configure(
api,
cipd_dir,
src_dir,
platform,
host,
[
'--disable-dependency-tracking',
'--disable-silent-rules',
'--disable-dtrace',
'--disable-libelf',
'--disable-libmount',
'--disable-shared',
'--enable-static',
'--prefix=',
'--with-pic',
'--with-pcre=internal',
] + extra_args,
cflags=('-I%s' % pkg_dir.join('include'),),
cxxflags=('-I%s' % pkg_dir.join('include'),),
ldflags=('-L%s' % pkg_dir.join('lib'),))
api.step('build', ['make', '-j%d' % api.goma.jobs])
api.step('install', ['make', 'install', 'DESTDIR=%s' % pkg_dir])
def build_qemu(api, cipd_dir, platform, host, revision, prod):
src_dir = api.path['start_dir'].join('qemu', 'src')
api.file.ensure_directory('qemu/src', src_dir)
repository = 'https://fuchsia.googlesource.com/third_party/qemu'
revision = api.git.checkout(
repository, src_dir, ref=revision, submodules=True)
build_dir = api.path['start_dir'].join('qemu', 'build')
api.file.ensure_directory('qemu/build', build_dir)
install_dir = api.path['start_dir'].join('qemu', 'install')
api.file.ensure_directory('qemu/install', install_dir)
target = PLATFORM_TO_TRIPLE[platform]
extra_options = {
'linux': [
'--cc=%s' % cipd_dir.join('bin', 'clang'),
'--cxx=%s' % cipd_dir.join('bin', 'clang++'),
'--build=%s' % PLATFORM_TO_TRIPLE[host],
'--host=%s' % target,
'--extra-cflags=--target=%s --sysroot=%s' %
(target, cipd_dir.join(platform)),
'--extra-cxxflags=--target=%s --sysroot=%s' %
(target, cipd_dir.join(platform)),
# Supress warning about the unused arguments because QEMU ignores
# --disable-werror at configure time which triggers an error because
# -static-libstdc++ is unused when linking C code.
'--extra-ldflags=--target=%s --sysroot=%s -static-libstdc++ -Qunused-arguments -lm -ldl -lpthread'
% (target, cipd_dir.join(platform)),
'--disable-gtk',
'--disable-x11',
'--enable-sdl',
'--enable-kvm',
],
'mac': ['--enable-cocoa',],
}[platform.split('-')[0]]
variables = {}
if platform.startswith('linux'):
variables.update({
'AR': cipd_dir.join('bin', 'llvm-ar'),
'RANLIB': cipd_dir.join('bin', 'llvm-ranlib'),
'NM': cipd_dir.join('bin', 'llvm-nm'),
'STRIP': cipd_dir.join('bin', 'llvm-strip'),
'OBJCOPY': cipd_dir.join('bin', 'llvm-objcopy'),
})
with api.context(cwd=build_dir, env=variables):
api.step('configure qemu', [
src_dir.join('configure'),
'--prefix=',
'--target-list=aarch64-softmmu,x86_64-softmmu',
'--without-system-fdt',
'--disable-attr',
'--disable-bluez',
'--disable-brlapi',
'--disable-bzip2',
'--disable-cap-ng',
'--disable-curl',
'--disable-debug-info',
'--disable-debug-tcg',
'--disable-docs',
'--disable-gcrypt',
'--disable-glusterfs',
'--disable-gnutls',
'--disable-guest-agent',
'--disable-libiscsi',
'--disable-libnfs',
'--disable-libssh2',
'--disable-libusb',
'--disable-libxml2',
'--disable-linux-aio',
'--disable-lzo',
'--disable-nettle',
'--disable-opengl',
'--disable-qom-cast-debug',
'--disable-rbd',
'--disable-rdma',
'--disable-seccomp',
'--disable-smartcard',
'--disable-snappy',
'--disable-spice',
'--disable-tasn1',
'--disable-tcg-interpreter',
'--disable-tcmalloc',
'--disable-tpm',
'--disable-usb-redir',
'--disable-vhost-scsi',
'--disable-vhost-vsock',
'--disable-virtfs',
'--disable-vnc-jpeg',
'--disable-vnc-png',
'--disable-vnc-sasl',
'--disable-vte',
'--disable-werror',
'--disable-xen',
'--enable-kvm',
] + extra_options)
api.step('build', ['make', '-j%d' % api.platform.cpu_count])
api.step('install', ['make', 'install', 'DESTDIR=%s' % install_dir])
qemu_version = api.file.read_text(
'version', src_dir.join('VERSION'), test_data='2.10.1')
assert qemu_version, 'Cannot determine QEMU version'
# Upload the installation to isolate.
isolated_hash = api.upload.upload_isolated(install_dir)
api.step.active_result.presentation.step_text = isolated_hash
if prod:
# Upload the installation to CIPD for production builds.
api.upload.cipd_package(
'fuchsia/third_party/qemu/' + platform,
install_dir, [api.upload.DirectoryPath(install_dir)],
{'git_revision': revision},
repository=repository)
def RunSteps(api, repository, branch, revision, platform, prod):
api.goma.ensure()
if not revision:
revision = api.gitiles.refs(repository).get(branch, None)
# TODO: factor this out into a host_build recipe module.
host_platform = '%s-%s' % (api.platform.name.replace('win', 'windows'), {
'intel': {
32: '386',
64: 'amd64',
},
'arm': {
32: 'armv6',
64: 'arm64',
},
}[api.platform.arch][api.platform.bits])
target_platform = platform or host_platform
with api.step.nest('ensure_packages'):
with api.context(infra_steps=True):
cipd_dir = api.path['start_dir'].join('cipd')
pkgs = api.cipd.EnsureFile()
pkgs.add_package('infra/cmake/${platform}', 'version:3.9.2')
pkgs.add_package('infra/ninja/${platform}', 'version:1.8.2')
pkgs.add_package('fuchsia/third_party/clang/${platform}', 'goma')
if target_platform.startswith('linux'):
pkgs.add_package(
'fuchsia/sysroot/linux-amd64',
'git_revision:a96053c799a0f1ad0b7e8ab8199edbfa18adcbb6',
'linux-amd64')
pkgs.add_package(
'fuchsia/sysroot/linux-arm64',
'git_revision:a96053c799a0f1ad0b7e8ab8199edbfa18adcbb6',
'linux-arm64')
pkgs.add_package('fuchsia/third_party/source/m4', 'version:1.4.18',
'source/m4')
pkgs.add_package('fuchsia/third_party/source/autoconf', 'version:2.69',
'source/autoconf')
pkgs.add_package('fuchsia/third_party/source/automake', 'version:1.16',
'source/automake')
pkgs.add_package('fuchsia/third_party/source/libtool', 'version:2.4.6',
'source/libtool')
pkgs.add_package('fuchsia/third_party/source/make', 'version:4.2.1',
'source/make')
pkgs.add_package('fuchsia/third_party/source/pkg-config', 'version:0.29',
'source/pkg-config')
pkgs.add_package('fuchsia/third_party/source/ncurses', 'version:6.1',
'source/ncurses')
pkgs.add_package('fuchsia/third_party/source/gettext', 'version:0.19.8.1',
'source/gettext')
pkgs.add_package('fuchsia/third_party/source/glib', 'version:2.58.2',
'source/glib')
pkgs.add_package('fuchsia/third_party/source/libffi', 'version:3.3-rc0',
'source/libffi')
pkgs.add_package('fuchsia/third_party/source/pixman', 'version:0.36.0',
'source/pixman')
pkgs.add_package('fuchsia/third_party/source/sdl', 'version:2.0.9',
'source/sdl')
pkgs.add_package('fuchsia/third_party/source/zlib', 'version:1.2.11',
'source/zlib')
api.cipd.ensure(cipd_dir, pkgs)
pkg_dir = api.path['start_dir'].join('pkgconfig')
api.file.ensure_directory('create pkg dir', pkg_dir)
variables = {
'PKG_CONFIG_SYSROOT_DIR':
pkg_dir,
'PKG_CONFIG_ALLOW_SYSTEM_CFLAGS':
1,
'PKG_CONFIG_ALLOW_SYSTEM_LIBS':
1,
'PKG_CONFIG_LIBDIR':
':'.join([
str(pkg_dir.join('share', 'pkgconfig')),
str(pkg_dir.join('lib', 'pkgconfig'))
]),
}
with api.context(env=variables, env_prefixes={'PATH': [pkg_dir.join('bin')]}):
with api.step.nest('make'):
build_make(api, cipd_dir, pkg_dir, host_platform, host_platform)
with api.step.nest('m4'):
build_m4(api, cipd_dir, pkg_dir, host_platform, host_platform)
with api.step.nest('autoconf'):
build_autoconf(api, cipd_dir, pkg_dir, host_platform, host_platform)
with api.step.nest('automake'):
build_automake(api, cipd_dir, pkg_dir, host_platform, host_platform)
with api.step.nest('libtool'):
build_libtool(api, cipd_dir, pkg_dir, host_platform, host_platform)
with api.step.nest('pkg-config'):
build_pkg_config(api, cipd_dir, pkg_dir, host_platform, host_platform)
if target_platform.startswith('linux'):
with api.step.nest('sdl'):
build_sdl(api, cipd_dir, pkg_dir, target_platform, host_platform)
with api.step.nest('zlib'):
build_zlib(api, cipd_dir, pkg_dir, target_platform, host_platform)
with api.step.nest('pixman'):
build_pixman(api, cipd_dir, pkg_dir, target_platform, host_platform)
with api.step.nest('libffi'):
build_libffi(api, cipd_dir, pkg_dir, target_platform, host_platform)
with api.step.nest('ncurses'):
build_ncurses(api, cipd_dir, pkg_dir, target_platform, host_platform)
if target_platform.startswith('mac'):
with api.step.nest('gettext'):
build_gettext(api, cipd_dir, pkg_dir, target_platform, host_platform)
with api.step.nest('glib'):
build_glib(api, cipd_dir, pkg_dir, target_platform, host_platform)
with api.step.nest('qemu'):
build_qemu(api, cipd_dir, target_platform, host_platform, revision, prod)
def GenTests(api):
revision = '75b05681239cb309a23fcb4f8864f177e5aa62da'
version = 'QEMU emulator version 2.8.0 (v2.8.0-15-g28cd8b6577-dirty)'
for platform in ['linux', 'mac']:
yield (api.test(platform) + api.platform.name(platform) +
api.gitiles.refs('refs', ('refs/heads/master', revision)) +
api.properties(
manifest='qemu',
remote='https://fuchsia.googlesource.com/manifest',
platform=platform + '-amd64') +
api.step_data('qemu.version', api.raw_io.stream_output(version)) +
api.post_process(StatusSuccess))
yield (api.test(platform + '_new') + api.platform.name(platform) +
api.gitiles.refs('refs', ('refs/heads/master', revision)) +
api.properties(
manifest='qemu',
remote='https://fuchsia.googlesource.com/manifest',
platform=platform + '-amd64') +
api.step_data('qemu.version', api.raw_io.stream_output(version)) +
api.step_data(
'qemu.cipd.cipd search fuchsia/third_party/qemu/' + platform +
'-amd64 ' + 'git_revision:deadbeef',
api.cipd.example_search('fuchsia/qemu/' + platform + '-amd64 ',
[])) + api.post_process(StatusSuccess))
yield (api.test('linux_arm64') + api.platform.name('linux') +
api.gitiles.refs('refs', ('refs/heads/master', revision)) +
api.properties(
manifest='qemu',
remote='https://fuchsia.googlesource.com/manifest',
platform='linux-arm64') +
api.step_data('qemu.version', api.raw_io.stream_output(version)) +
api.post_process(StatusSuccess))