blob: 0d00a4f4fd5c0db73cb234f53630df14d7dcdfc7 [file] [log] [blame]
# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building QEMU."""
import shlex
from PB.recipes.fuchsia.qemu import InputProperties
DEPS = [
"fuchsia/cas_util",
"fuchsia/cipd_util",
"fuchsia/cmake",
"fuchsia/git_checkout",
"fuchsia/goma",
"fuchsia/linux_sdk",
"fuchsia/macos_sdk",
"fuchsia/ninja",
"fuchsia/platform_util",
"fuchsia/zlib",
"recipe_engine/cipd",
"recipe_engine/context",
"recipe_engine/file",
"recipe_engine/path",
"recipe_engine/platform",
"recipe_engine/properties",
"recipe_engine/raw_io",
"recipe_engine/step",
]
PROPERTIES = InputProperties
def RunSteps(api, props):
checkout_dir, revision = api.git_checkout(
props.remote,
fallback_ref=props.ref or "refs/heads/master",
submodules=True,
config={
"url.https://qemu.googlesource.com/.insteadOf": "https://gitlab.com/qemu-project/"
},
)
qemu_version = api.file.read_text(
"read qemu version", checkout_dir.join("VERSION")
).strip()
host_platform = api.platform_util.host
target_platform = api.platform_util.platform(
props.platform or host_platform.platform
)
with api.step.nest("ensure_packages"), api.context(infra_steps=True):
cipd_dir = api.path["start_dir"].join("cipd")
pkgs = api.cipd.EnsureFile()
pkgs.add_package("fuchsia/third_party/clang/${platform}", "integration")
pkgs.add_package("fuchsia/third_party/autoconf/${platform}", "version:2.69")
pkgs.add_package("fuchsia/third_party/automake/${platform}", "version:1.16.2")
pkgs.add_package("fuchsia/third_party/libtool/${platform}", "version:2.4.6")
pkgs.add_package("fuchsia/third_party/m4/${platform}", "version:1.4.18")
pkgs.add_package("fuchsia/third_party/make/${platform}", "version:4.3")
pkgs.add_package("fuchsia/third_party/pkg-config/${platform}", "version:0.29.2")
pkgs.add_package(
"fuchsia/third_party/source/gettext",
"version:0.19.8.1",
"source/gettext",
)
pkgs.add_package(
"fuchsia/third_party/source/glib", "version:2.58.2", "source/glib"
)
pkgs.add_package(
"fuchsia/third_party/source/libffi", "version:3.3", "source/libffi"
)
pkgs.add_package(
"fuchsia/third_party/source/pixman", "version:0.36.0", "source/pixman"
)
pkgs.add_package(
"fuchsia/third_party/source/sdl", "version:2.0.12", "source/sdl"
)
api.cipd.ensure(cipd_dir, pkgs)
pkg_dir = api.path["start_dir"].join("pkgconfig")
api.file.ensure_directory("create pkg dir", pkg_dir)
variables = {
"PKG_CONFIG_PATH": "",
# TODO: We should be using the following but that breaks the SDL dependency.
# "PKG_CONFIG_SYSROOT_DIR": platform_sysroot(api, target_platform),
"PKG_CONFIG_SYSROOT_DIR": pkg_dir,
"PKG_CONFIG_ALLOW_SYSTEM_CFLAGS": 0,
"PKG_CONFIG_ALLOW_SYSTEM_LIBS": 0,
"PKG_CONFIG_LIBDIR": ":".join(
[
str(pkg_dir.join("share", "pkgconfig")),
str(pkg_dir.join("lib", "pkgconfig")),
]
),
}
with api.linux_sdk(), api.macos_sdk(), api.context(
env=variables,
env_prefixes={"PATH": [cipd_dir.join("bin"), pkg_dir.join("bin")]},
):
if target_platform.is_linux:
with api.step.nest("sdl"):
api.cmake.build_with_ninja(
src_dir=cipd_dir.join("source", "sdl"),
extra_args=cmake_args(api, cipd_dir, target_platform)
+ [
"-DVIDEO_WAYLAND=OFF",
"-DSDL_SHARED=OFF",
"-DSDL_STATIC_PIC=ON",
"-DGCC_ATOMICS=ON",
],
install_dir=pkg_dir,
)
with api.step.nest("zlib"):
api.zlib.build(
cmake_extra_args=cmake_args(api, cipd_dir, target_platform),
platform=target_platform,
install_dir=pkg_dir,
)
with api.step.nest("pixman"):
build_pixman(
api, cipd_dir, pkg_dir, target_platform, host_platform, props.prod
)
with api.step.nest("libffi"):
build_libffi(
api, cipd_dir, pkg_dir, target_platform, host_platform, props.prod
)
with api.step.nest("gettext"):
build_gettext(
api, cipd_dir, pkg_dir, target_platform, host_platform, props.prod
)
with api.step.nest("glib"):
build_glib(
api, cipd_dir, pkg_dir, target_platform, host_platform, props.prod
)
with api.step.nest("qemu"):
install_dir = build_qemu(
api,
checkout_dir,
pkg_dir,
cipd_dir,
target_platform,
host_platform,
props.prod,
)
api.cas_util.upload(install_dir, output_property="isolated")
if props.prod:
api.cipd_util.upload_package(
f"fuchsia/third_party/qemu/{target_platform.platform}",
install_dir,
search_tag={"git_revision": revision},
repository=props.remote,
metadata=[("version", qemu_version)],
)
def cmake_args(api, cipd_dir, platform):
# TODO(b/270963251): Enable optimizations for cmake.
args = [
f"-DCMAKE_MAKE_PROGRAM={api.ninja.path}",
f"-DCMAKE_C_COMPILER={cipd_dir.join('bin', 'clang')}",
f"-DCMAKE_C_COMPILER_TARGET={platform.triple}",
f"-DCMAKE_CXX_COMPILER={cipd_dir.join('bin', 'clang++')}",
f"-DCMAKE_CXX_COMPILER_TARGET={platform.triple}",
f"-DCMAKE_SYSROOT={platform_sysroot(api, platform)}",
"-DCMAKE_INSTALL_PREFIX=",
f"-DCMAKE_LINKER={'/usr/bin/ld' if platform.is_mac else cipd_dir.join('bin', 'ld.lld')}",
f"-DCMAKE_NM={cipd_dir.join('bin', 'llvm-nm')}",
f"-DCMAKE_OBJCOPY={cipd_dir.join('bin', 'llvm-objcopy')}",
f"-DCMAKE_OBJDUMP={cipd_dir.join('bin', 'llvm-objdump')}",
f"-DCMAKE_RANLIB={cipd_dir.join('bin', 'llvm-ranlib')}",
f"-DCMAKE_STRIP={cipd_dir.join('bin', 'llvm-strip')}",
]
return args
def platform_sysroot(api, platform):
if platform.is_linux:
return api.linux_sdk.sysroot
if platform.is_mac:
return api.macos_sdk.sysroot
raise api.step.StepFailure("supported platform") # pragma: no cover
def environment(
api, cipd_dir, platform, host, optimize, cflags=(), cxxflags=(), ldflags=()
):
sysroot = [f"--sysroot={platform_sysroot(api, platform)}"]
target = (
[f"--target={platform.triple}"] if platform.platform != host.platform else []
)
opt = []
if optimize:
opt = ["-O3"]
# TODO: Enable LTO for Mac targets.
#
# On Mac this currently fails basic Make compile checks
# configure: error: in `/opt/s/w/ir/x/w/make':
# configure: error: C compiler cannot create executables
if platform.is_linux:
opt.extend(["-flto", "-fwhole-program-vtables"])
variables = {
"CC": cipd_dir.join("bin", "clang"),
"CXX": cipd_dir.join("bin", "clang++"),
"CFLAGS": " ".join(sysroot + target + opt + list(cflags)),
"CPPFLAGS": " ".join(sysroot + target + opt + list(cflags)),
"CXXFLAGS": " ".join(sysroot + target + opt + list(cxxflags)),
"LDFLAGS": " ".join(sysroot + target + opt + list(ldflags)),
"AR": cipd_dir.join("bin", "llvm-ar"),
"RANLIB": cipd_dir.join("bin", "llvm-ranlib"),
"NM": cipd_dir.join("bin", "llvm-nm"),
"STRIP": cipd_dir.join("bin", "llvm-strip"),
"OBJCOPY": cipd_dir.join("bin", "llvm-objcopy"),
}
return variables
def configure(
api,
cipd_dir,
src_dir,
platform,
host,
optimize,
flags=(),
cflags=(),
cxxflags=(),
ldflags=(),
step_name="configure",
):
variables = environment(
api, cipd_dir, platform, host, optimize, cflags, cxxflags, ldflags
)
if platform.platform != host.platform:
flags.extend(
[
# config.sub used by QEMU doesn't yet recognize arm64-apple-darwin which
# is used by Clang, so replace it with aarch64-apple-darwin to make the
# script not fail.
f"--build={host.triple.replace('arm64-apple-darwin', 'aarch64-apple-darwin')}",
f"--host={platform.triple.replace('arm64-apple-darwin', 'aarch64-apple-darwin')}",
]
)
return api.step(
step_name,
[src_dir.join("configure")]
+ flags
+ sorted(f"{k}={v}" for k, v in variables.items()),
)
def build_pixman(api, cipd_dir, pkg_dir, platform, host, optimize):
src_dir = cipd_dir.join("source", "pixman")
build_dir = api.path["start_dir"].join("pixman")
api.file.ensure_directory("pixman", build_dir)
with api.context(cwd=build_dir):
configure(
api,
cipd_dir,
src_dir,
platform,
host,
optimize,
[
"--disable-dependency-tracking",
"--disable-gtk",
"--disable-shared",
"--disable-silent-rules",
"--enable-static",
"--prefix=",
"--with-pic",
],
)
# pixman tests do not compile under LTO due to missing symbols [0]. We
# get around this by only building/installing the pixman directory,
# then dropping then installing the `pc` files so the library can be
# found. We could also fix this by upreving pixman to a new version
# which supports skipping test/demo builds natively.
#
# Missing symbol example:
#
# ld.lld: error: undefined symbol: __floatditf
# >>> referenced by ld-temp.o
# >>> lto.tmp:(test_matrix)
# >>> referenced by ld-temp.o
# >>> lto.tmp:(test_matrix)
# >>> referenced by ld-temp.o
# >>> lto.tmp:(test_matrix)
# >>> referenced 3 more times
#
# Adding `--rtlib=compiler-rt` to LDFLAGS didn't seem to help, maybe I
# misunderstand where that symbol is supposed to come from.
api.step("build", ["make", "-C", "pixman", f"-j{int(api.goma.jobs)}"])
api.step("install", ["make", "install", "-C", "pixman", f"DESTDIR={pkg_dir}"])
api.step(
"install pixman-1.pc",
["make", "install-pkgconfigDATA", f"DESTDIR={pkg_dir}"],
)
def build_libffi(api, cipd_dir, pkg_dir, platform, host, optimize):
src_dir = cipd_dir.join("source", "libffi")
build_dir = api.path["start_dir"].join("libffi")
api.file.ensure_directory("libffi", build_dir)
with api.context(cwd=build_dir):
configure(
api,
cipd_dir,
src_dir,
platform,
host,
optimize,
[
"--disable-debug",
"--disable-dependency-tracking",
"--disable-docs",
"--disable-shared",
"--enable-static",
"--prefix=",
"--target=%s"
# config.sub used by QEMU doesn't yet recognize arm64-apple-darwin which
# is used by Clang, so replace it with aarch64-apple-darwin to make the
# script not fail.
% platform.triple.replace("arm64-apple-darwin", "aarch64-apple-darwin"),
"--with-pic",
],
)
api.step("build", ["make", f"-j{int(api.goma.jobs)}"])
api.step("install", ["make", "install", f"DESTDIR={pkg_dir}"])
def build_gettext(api, cipd_dir, pkg_dir, platform, host, optimize):
# Build gettext-{runtime,tools} separately, when cross-compiling we need to
# always build `tools` for the host and the `runtime` for the target. This
# is how the PACKAGING file recommends we split them.
#
# The order of tools, runtime is important here; the tools build will
# overwrite the runtimes installation.
for package in ["gettext-tools", "gettext-runtime"]:
src_dir = cipd_dir.join("source", "gettext", package)
build_dir = api.path["start_dir"].join("gettext", package)
api.file.ensure_directory("gettext", build_dir)
with api.context(cwd=build_dir):
configure(
api,
cipd_dir,
src_dir,
platform if package == "gettext-runtime" else host,
host,
optimize,
[
"--disable-curses",
"--disable-dependency-tracking",
"--disable-silent-rules",
"--disable-debug",
"--disable-shared",
"--disable-java",
"--disable-csharp",
"--disable-c++",
"--disable-openmp",
"--enable-static",
"--prefix=",
"--with-pic",
"--with-included-gettext",
"--with-included-glib",
"--with-included-libcroco",
"--with-included-libunistring",
"--without-git",
"--without-cvs",
"--without-xz",
],
ldflags=["-lm"],
)
api.step("build", ["make", f"-j{int(api.goma.jobs)}"])
api.step("install", ["make", "install", f"DESTDIR={pkg_dir}"])
def build_glib(api, cipd_dir, pkg_dir, platform, host, optimize):
src_dir = cipd_dir.join("source", "glib")
build_dir = api.path["start_dir"].join("glib")
api.file.ensure_directory("glib", build_dir)
extra_args = []
if platform.platform != host.platform:
api.file.write_text(
"cache",
build_dir.join(f"{platform}.cache"),
"""
glib_cv_long_long_format=ll
glib_cv_stack_grows=no
glib_cv_uscore=no
""",
)
extra_args.append(f"--cache-file={platform}.cache")
with api.context(
cwd=src_dir, env={"NOCONFIGURE": "1", "M4": cipd_dir.join("bin", "m4")}
):
api.step("autogen", [src_dir.join("autogen.sh")])
with api.context(cwd=build_dir):
configure(
api,
cipd_dir,
src_dir,
platform,
host,
optimize,
[
"--disable-dependency-tracking",
"--disable-silent-rules",
"--disable-dtrace",
"--disable-libelf",
"--disable-libmount",
"--disable-shared",
"--enable-static",
"--prefix=",
"--with-pic",
"--with-pcre=internal",
]
+ extra_args,
cflags=[
f"-I{pkg_dir.join('include')}",
"-Wno-declaration-after-statement",
"-Wno-int-conversion",
],
cxxflags=[f"-I{pkg_dir.join('include')}"],
ldflags=[f"-L{pkg_dir.join('lib')}"],
)
api.step("build", ["make", f"-j{int(api.goma.jobs)}"])
api.step("install", ["make", "install", f"DESTDIR={pkg_dir}"])
def build_qemu(
api,
checkout_dir,
pkg_dir,
cipd_dir,
target_platform,
host_platform,
optimize,
):
"""
Build QEMU.
Args:
checkout_dir (Path): Path to QEMU Git repository.
cipd_dir (Path): Path to prepopulated CIPD package dependencies.
target_platform (Platform): Target platform.
host_platform (Platform): Host platform.
Returns:
Path: Path to install directory.
"""
build_dir = api.path["start_dir"].join("qemu", "build")
api.file.ensure_directory("qemu/build", build_dir)
install_dir = api.path["start_dir"].join("qemu", "install")
api.file.ensure_directory("qemu/install", install_dir)
sysroot = [f"--sysroot={platform_sysroot(api, target_platform)}"]
target = (
[f"--target={target_platform.triple}"]
if target_platform.platform != host_platform.platform
else []
)
# TODO: QEMU builds fails to find zlib headers without an explicit -I which
# is a bug.
includes = ["-I%s" % pkg_dir.join("include")]
opt = []
if optimize:
opt = ["-O3"]
# TODO: Enable LTO for Mac targets.
if target_platform.is_linux:
opt.extend(["-flto", "-fwhole-program-vtables"])
extra_options = {
"linux": [
f"--build={host_platform.triple}",
f"--host={target}",
f"--extra-cflags={' '.join(sysroot + opt + target + includes)}",
f"--extra-cxxflags={' '.join(sysroot + opt + target + includes)}",
# Suppress warning about the unused arguments because QEMU ignores
# --disable-werror at configure time which triggers an error because
# -static-libstdc++ is unused when linking C code.
f"--extra-ldflags={' '.join(sysroot + target + opt + ['-ldl -static-libstdc++ -Qunused-arguments'])}",
"--disable-gtk",
"--enable-sdl",
"--enable-kvm",
],
"mac": [
f"--objcc={cipd_dir.join('bin', 'clang')}",
"--enable-cocoa",
f"--extra-cflags={' '.join(sysroot + target + includes)}",
f"--extra-cxxflags={' '.join(sysroot + target + includes)}",
"--extra-ldflags=%s"
% " ".join(
sysroot + target + ["-nostdlib++ %s" % cipd_dir.join("lib", "libc++.a")]
),
"--enable-hvf",
],
}[target_platform.platform.split("-")[0]]
if target_platform.platform != host_platform.platform:
extra_options.append("--cross-prefix=")
# NOTE: We don't want to pass --static to configure since we still link libc
# as a shared library, but we want everything else to use static linking.
variables = {
"AR": cipd_dir.join("bin", "llvm-ar"),
"RANLIB": cipd_dir.join("bin", "llvm-ranlib"),
"NM": cipd_dir.join("bin", "llvm-nm"),
"STRIP": cipd_dir.join("bin", "llvm-strip"),
"OBJCOPY": cipd_dir.join("bin", "llvm-objcopy"),
"QEMU_PKG_CONFIG_FLAGS": "--static",
}
if target_platform.is_mac:
variables.update(
{
# A bug in Meson treats the default linker of ld64.lld as a GNU-style linker;
# explicitly override it to use ld from the xcode installation.
"CC_LD": ld_path(api),
# TODO(joshuaseaton): Pass via --extra-objcflags once
# https://patchwork.kernel.org/project/qemu-devel/patch/20220215080307.69550-3-f4bug@amsat.org/
# is mainlined. "OBJCFLAGS" is something known by meson itself, but not the
# QEMU project.
"OBJCFLAGS": " ".join(sysroot + target),
}
)
with api.context(cwd=build_dir, env=variables):
try:
api.step(
"configure qemu",
[
checkout_dir.join("configure"),
f"--cc={cipd_dir.join('bin', 'clang')}",
# See function docstring for justification and associated bug.
f"--cxx={wrap_clang(api, target_platform, cipd_dir.join('bin', 'clang++'))}",
"--disable-attr",
"--disable-auth-pam",
"--disable-bochs",
"--disable-brlapi",
"--disable-bzip2",
"--disable-cap-ng",
"--disable-cloop",
"--disable-curl",
"--disable-debug-info",
"--disable-debug-tcg",
"--disable-dmg",
"--disable-docs",
"--disable-gcrypt",
"--disable-glusterfs",
"--disable-gnutls",
"--disable-guest-agent",
"--disable-iconv",
"--disable-libiscsi",
"--disable-libnfs",
"--disable-libpmem",
"--disable-libusb",
"--disable-linux-aio",
"--disable-lzo",
"--disable-nettle",
"--disable-opengl",
"--disable-parallels",
"--disable-plugins",
"--disable-qcow1",
"--disable-qed",
"--disable-qom-cast-debug",
"--disable-rbd",
"--disable-rdma",
"--disable-sdl-image",
"--disable-seccomp",
"--disable-smartcard",
"--disable-snappy",
"--disable-spice",
"--disable-tcg-interpreter",
"--disable-usb-redir",
"--disable-vdi",
"--disable-vhost-user",
"--disable-virtfs",
"--disable-vnc-jpeg",
"--disable-vnc-sasl",
"--disable-vte",
"--disable-werror",
"--disable-xen",
"--enable-fdt=git", # Build libfdt from scratch.
"--enable-vvfat", # For EFI support.
f"--ninja={api.ninja.path}",
f"--prefix={install_dir}",
"--target-list=aarch64-softmmu,arm-softmmu,riscv64-softmmu,x86_64-softmmu",
]
+ extra_options,
)
finally:
try:
api.file.read_text(
"meson-log.txt",
build_dir.join("meson-logs", "meson-log.txt"),
test_data="The Meson build system\nVersion: 1.0.0",
)
except: # pragma: no cover
pass
# config-host.h is a generated header consisting of #-defines detailing
# configuration settings, including whether certain features are
# enabled. It serves as an important diagnostic.
api.file.read_text(
"config-host.h",
build_dir.join("config-host.h"),
test_data="#pragma once\n#define CONFIG_FOO\n#undef CONFIG_BAR",
)
api.step("build", ["make", f"-j{int(api.platform.cpu_count)}"])
api.step("install", ["make", "install"])
# TODO(fxbug.dev/106763): Re-enable on Linux when passing again.
if (
target_platform.platform == host_platform.platform
and target_platform.is_mac
):
# V=1, DEBUG=1 for more informative test output.
api.step("run unit tests", ["make", "V=1", "DEBUG=1", "check"])
return install_dir
def ld_path(api):
return api.step(
"find ld",
["xcrun", "-f", "ld"],
stdout=api.raw_io.output_text(name="ld path", add_output_log=True),
step_test_data=lambda: api.raw_io.test_api.stream_output_text(
"/some/xcode/path/usr/bin/ld"
),
).stdout.strip()
# TODO(https://github.com/mesonbuild/meson/issues/8592): Meson links against
# libstdc++ by default when using clang. We use libc++ and so need to filter
# that out.
#
# TODO(https://github.com/mesonbuild/meson/issues/6377): Meson doesn't fully
# respect the linker override above (`CC_LD`). On top of that, specifying
# of that, specifying ld through `--extra-ldflags` alone appears to be
# insufficient to specify the linker everywhere. As a result, we're forced to
# bake in the specification into a wrapper.
def wrap_clang(api, platform, clang):
default_args = []
if platform.is_mac:
default_args.append(f"--ld-path={ld_path(api)}")
contents = """#!/bin/sh
args=("$@")
args=("${args[@]/-lstdc++}")
exec %s %s "${args[@]}"
""" % (
shlex.join(default_args),
clang,
)
wrapper = api.path["tmp_base"].join("clang-wrapper.sh")
api.file.write_text(f"write {api.path.basename(wrapper)}", wrapper, contents)
api.step(
f"make {api.path.basename(wrapper)} executable",
["chmod", "+x", wrapper],
)
return wrapper
def GenTests(api):
version = "8.0.50"
def properties(platform, prod=True):
return api.properties(
remote="https://fuchsia.googlesource.com/third_party/qemu",
platform=platform,
prod=prod,
)
yield (
api.test("linux-amd64")
+ api.platform.name("linux")
+ properties("linux-amd64")
+ api.step_data("read qemu version", api.file.read_text(version))
)
yield (
api.test("mac-amd64")
+ api.platform.name("mac")
+ properties("mac-amd64")
+ api.step_data("read qemu version", api.file.read_text(version))
)
yield (
api.test("linux-arm64")
+ api.platform.name("linux")
+ properties("linux-arm64", prod=False)
+ api.step_data("read qemu version", api.file.read_text(version))
)