[build][scripts] Integrate Zircon via new GN build rather than old make build
build-zircon.sh and thus `fx build-zircon` now runs gn (if necessary) and
ninja (just once) rather than running `make` several times. The extra
arguments previously taken as make arguments are now taken as either GN
build arguments or ninja arguments.
`fx set` now runs Zircon's `gn gen` before Fuchsia's `gn gen`, which
consumes files generated by the former.
Zircon-related GN build arguments are gone. Zircon configuration
can be controlled via arguments to build-zircon.sh instead.
Note this also means that `zircon_boot_groups="all"` behavior is now
always on. That is, the BOOTFS in the Fuchsia boot images will be
fatter and contain all of the Zircon tools and tests in `/boot`.
Zircon tests now appear in `/boot/test` rather than `/system/test`.
Bug: BLD-325 #comment Fuchsia GN integration adapted to Zircon GN legacy support
Test: manual
Change-Id: I6e0b49b65c78a27e71078d32e40d0b22d366d666
diff --git a/build/config/fuchsia/BUILD.gn b/build/config/fuchsia/BUILD.gn
index 4864667..2580aa0c 100644
--- a/build/config/fuchsia/BUILD.gn
+++ b/build/config/fuchsia/BUILD.gn
@@ -114,12 +114,16 @@
}
config("fdio_config") {
- libs = [ "fdio" ]
-
# TODO(pylaligand): find a better way to let executables link in fdio.
# Ideally their dependencies should be set up in such a way that it would get
# inherited from them.
- lib_dirs = [ "$zircon_build_dir/system/ulib/fdio" ]
+ foreach(image, zircon_images) {
+ if (image.name == "fdio" && image.cpu == current_cpu) {
+ assert(image.type == "so")
+ libs = [ "$zircon_root_build_dir/${image.path}" ]
+ }
+ }
+ assert(defined(libs), "fdio.so missing from Zircon images.json")
}
config("executable_config") {
diff --git a/build/config/fuchsia/zircon.gni b/build/config/fuchsia/zircon.gni
index 27a700e1..75064ab 100644
--- a/build/config/fuchsia/zircon.gni
+++ b/build/config/fuchsia/zircon.gni
@@ -4,59 +4,19 @@
import("//build/config/clang/clang.gni")
-declare_args() {
- # Where to find Zircon's host-side tools that are run as part of the build.
- zircon_tools_dir = "//out/build-zircon/tools"
+# This is $root_build_dir in the Zircon GN build.
+zircon_root_build_dir = "//out/build-zircon"
- # Zircon build directory for `target_cpu`, containing link-time `.so.abi`
- # files that GN `deps` on //zircon/public libraries will link against.
- # This should not be a sanitizer build.
- zircon_build_abi_dir = "//out/build-zircon/build-$target_cpu"
+# The top-level `tools` Ninja target in Zircon puts the tool binaries here.
+zircon_tools_dir = "$zircon_root_build_dir/tools"
- # Zircon build directory for `target_cpu`, containing `.manifest` and
- # `.zbi` files for Zircon's BOOTFS and kernel. This provides the kernel
- # and Zircon components used in the boot image. It also provides the
- # Zircon shared libraries used at runtime in Fuchsia packages.
- #
- # If left `""` (the default), then this is computed from
- # [`zircon_build_abi_dir`](#zircon_build_abi_dir) and
- # [`zircon_use_asan`](#zircon_use_asan).
- zircon_build_dir = ""
+# The `gn gen` stage of the Zircon GN build writes this file.
+# It's a list of {name=... path=... type=...} scopes.
+zircon_images = read_file("$zircon_root_build_dir/images.json", "json")
- # Zircon `USE_ASAN=true` build directory for `target_cpu` containing
- # `bootfs.manifest` with libraries and `devhost.asan`.
- #
- # If left `""` (the default), then this is computed from
- # [`zircon_build_dir`](#zircon_build_dir) and
- # [`zircon_use_asan`](#zircon_use_asan).
- zircon_asan_build_dir = ""
-
- # Set this if [`zircon_build_dir`](#zircon_build_dir) was built with
- # `USE_ASAN=true`, e.g. `//scripts/build-zircon.sh -A`. This mainly
- # affects the defaults for [`zircon_build_dir`](#zircon_build_dir) and
- # [`zircon_build_abi_dir`](#zircon_build_abi_dir). It also gets noticed
- # by //scripts/fx commands that rebuild Zircon so that they use `-A`
- # again next time.
- zircon_use_asan = false
-
- # Path to `make` binary. By default, `make` is assumed to be in the
- # path. Used in the script that generates the Zircon build rules. N.B. this
- # path is *not* rebased, just used as is.
- zircon_make_path = "make"
-}
-
-if (zircon_build_dir == "") {
- zircon_build_dir = zircon_build_abi_dir
- if (zircon_use_asan) {
- zircon_build_dir += "-asan"
- }
-}
-
-if (zircon_asan_build_dir == "") {
- if (zircon_use_asan) {
- zircon_asan_build_dir = zircon_build_dir
- } else {
- zircon_asan_build_dir = "${zircon_build_dir}-asan"
+foreach(image, zircon_images) {
+ if (image.name == "kernel" && image.type == "zbi" && image.cpu == target_cpu) {
+ zircon_kernel_zbi = "$zircon_root_build_dir/${image.path}"
}
}
diff --git a/build/gn/BUILD.gn b/build/gn/BUILD.gn
index 26b64c6..3b74bcc 100644
--- a/build/gn/BUILD.gn
+++ b/build/gn/BUILD.gn
@@ -112,14 +112,10 @@
[
"--out",
rebase_path("//zircon/public"),
- "--staging",
- rebase_path("$root_out_dir/zircon-gn"),
- "--zircon-user-build",
- rebase_path(zircon_build_abi_dir),
- "--zircon-tool-build",
- rebase_path("$zircon_tools_dir/.."),
- "--make",
- zircon_make_path,
+ "--zircon-build",
+ rebase_path(zircon_root_build_dir),
+ "--zircon-manifest",
+ rebase_path("$zircon_root_build_dir/export/manifest-$target_cpu"),
],
"",
zircon_files + supporting_templates)
@@ -134,24 +130,6 @@
"FUCHSIA_BUILD_DIR='${_relative_build_dir}'",
"FUCHSIA_ARCH='${target_cpu}'",
]
-if (use_goma) {
- _fx_config_lines += [
- "# This will affect Zircon's make via //scripts/build-zircon.sh.",
- "export GOMACC='${goma_dir}/gomacc'",
- ]
-}
-_fx_build_zircon_args = ""
-if (zircon_use_asan) {
- _fx_build_zircon_args += " -A"
-}
-foreach(selector, select_variant) {
- if (selector == "host_asan") {
- _fx_build_zircon_args += " -H"
- }
-}
-if (_fx_build_zircon_args != "") {
- _fx_config_lines += [ "FUCHSIA_BUILD_ZIRCON_ARGS=($_fx_build_zircon_args)" ]
-}
write_file("$root_build_dir/fx.config", _fx_config_lines)
# Generates breakpad symbol data for unstripped binaries.
diff --git a/build/images/BUILD.gn b/build/images/BUILD.gn
index e0f790f..4458773 100644
--- a/build/images/BUILD.gn
+++ b/build/images/BUILD.gn
@@ -19,40 +19,11 @@
import("//garnet/go/src/pm/pm.gni")
declare_args() {
- # Groups to include from the Zircon /boot manifest into /boot.
- # This is either "all" or a comma-separated list of one or more of:
- # core -- necessary to boot
- # misc -- utilities in /bin
- # test -- test binaries in /bin and /test
- zircon_boot_groups = "core"
-
# Path to manifest file containing data to place into the initial /data
# partition.
data_partition_manifest = ""
}
-declare_args() {
- # Groups to include from the Zircon /boot manifest into /system
- # (instead of into /boot like Zircon's own bootdata.bin does).
- # Should not include any groups that are also in zircon_boot_groups,
- # which see. If zircon_boot_groups is "all" then this should be "".
- # **TODO(mcgrathr)**: _Could default to "" for `!is_debug`, or "production
- # build". Note including `"test"` here places all of Zircon's tests into
- # `/system/test`, which means that Fuchsia bots run those tests too._
- zircon_system_groups = "misc,test"
- if (zircon_boot_groups == "all") {
- zircon_system_groups = ""
- }
-}
-
-if (zircon_boot_groups == "all") {
- assert(zircon_system_groups == "",
- "zircon_boot_groups already has everything")
-} else {
- assert(zircon_system_groups != "all" && zircon_system_groups != "core",
- "zircon_system_groups cannot include core (or all)")
-}
-
# This will collect a list of scopes describing each image exported.
# See json.gni.
images = [
@@ -290,10 +261,10 @@
# /system manifest files can assume that the /boot files are visible at
# runtime, so dependencies already in /boot won't be copied into /system.
bootfs_manifest = boot_manifest
- bootfs_zircon_groups = zircon_boot_groups
+ bootfs_zircon_groups = "all"
# Collect whatever we want from Zircon that didn't go into /boot.
- zircon_groups = zircon_system_groups
+ zircon_groups = ""
# Now each package() target in the build contributes manifest entries.
# For system_image packages, these contain binaries that need their
@@ -469,14 +440,12 @@
}
sdk = "qemu-kernel.bin"
deps = []
- if (current_cpu == "arm64") {
- sources = [
- "$zircon_build_dir/qemu-boot-shim.bin",
- ]
- } else if (current_cpu == "x64") {
- sources = [
- "$zircon_build_dir/multiboot.bin",
- ]
+ foreach(image, zircon_images) {
+ if (image.name == "qemu-kernel" && image.cpu == target_cpu) {
+ sources = [
+ "$zircon_root_build_dir/${image.path}",
+ ]
+ }
}
},
]
@@ -563,7 +532,7 @@
":system_image.manifest",
]
inputs = [
- "${zircon_build_dir}/kernel.zbi",
+ zircon_kernel_zbi,
boot_manifest,
]
manifest = [
@@ -1502,6 +1471,9 @@
###
### Build ID maps.
###
+### TODO(TC-303): ids.txt is deprecated and will be removed.
+### The toolchain rules already populate $root_build_dir/.build-id/.
+###
# Combine the /boot, /system, and package build ID maps into one.
# Nothing in the build uses this, but top-level targets always update
@@ -1512,8 +1484,6 @@
deps = [
":kernel-ids.txt",
":system_image.manifest",
- ":zircon-asan-build-id",
- ":zircon-build-id",
]
sources = [
"$target_out_dir/kernel-ids.txt",
@@ -1549,7 +1519,7 @@
action("kernel-ids.txt") {
script = "manifest.py"
sources = [
- "$zircon_build_dir/ids.txt",
+ "$zircon_root_build_dir/ids-$current_cpu.txt",
]
outputs = [
"$target_out_dir/kernel-ids.txt",
@@ -1557,45 +1527,12 @@
args = [
"--separator= ",
"--output=" + rebase_path(outputs[0], root_build_dir),
- "--include-source=*/libzircon.so",
+ "--include-source=*/libzircon.so.debug",
"--include-source=*/zircon.elf",
"--manifest=" + rebase_path(sources[0], root_build_dir),
]
}
-# TODO(TC-303): This is a temporary hack to get all of Zircon's debug files
-# into the $root_build_dir/.build-id hierarchy. The Zircon build produces
-# its own .build-id hierarchy under $zircon_build_dir, but using its ids.txt
-# is the simpler way to populate the one in this build. When ids.txt is fully
-# obsolete, hopefully Zircon will be in the unified build anyway.
-foreach(target,
- [
- {
- name = "zircon-build-id"
- sources = [
- "$zircon_build_dir/ids.txt",
- ]
- },
- {
- name = "zircon-asan-build-id"
- sources = [
- "$zircon_asan_build_dir/ids.txt",
- ]
- },
- ]) {
- action(target.name) {
- visibility = [ ":ids.txt" ]
- script = "//scripts/build_id_conv.py"
- sources = target.sources
- outputs = [
- "$root_build_dir/${target_name}.stamp",
- ]
- args =
- [ "--stamp=" + rebase_path(outputs[0], root_build_dir) ] +
- rebase_path(sources + [ "$root_build_dir/.build-id" ], root_build_dir)
- }
-}
-
images += [
{
deps = [
diff --git a/build/images/boot.gni b/build/images/boot.gni
index f66f0e8..07c6905 100644
--- a/build/images/boot.gni
+++ b/build/images/boot.gni
@@ -65,7 +65,11 @@
# The Multiboot trampoline is the "kernel" (`--vmlinuz` switch) and the
# ZBI is the RAM disk (`--bootloader` switch).
assert(current_cpu == "x64")
- kernel = "${zircon_build_dir}/multiboot.bin"
+ foreach(image, zircon_images) {
+ if (image.name == "multiboot-kernel" && image.cpu == target_cpu) {
+ kernel = "$zircon_root_build_dir/${image.path}"
+ }
+ }
inputs += [ kernel ]
args = [
@@ -182,7 +186,12 @@
}
if (target_cpu == "x64") {
- gigaboot_bin = "${zircon_build_dir}/bootloader/bootx64.efi"
+ foreach(image, zircon_images) {
+ if (image.name == "bootloader" && image.type == "efi" &&
+ image.cpu == target_cpu) {
+ gigaboot_bin = "$zircon_root_build_dir/${image.path}"
+ }
+ }
args += [
"--efi-bootloader",
rebase_path(gigaboot_bin),
diff --git a/build/images/custom_signing.gni b/build/images/custom_signing.gni
index 6a895a4..ff8085e 100644
--- a/build/images/custom_signing.gni
+++ b/build/images/custom_signing.gni
@@ -93,7 +93,7 @@
"-v",
rebase_path(vbmeta_file, root_build_dir),
"-B",
- rebase_path(zircon_build_dir, root_build_dir),
+ rebase_path(zircon_root_build_dir, root_build_dir),
]
}
}
diff --git a/build/images/finalize_manifests.py b/build/images/finalize_manifests.py
index cb05b79..0163dae 100755
--- a/build/images/finalize_manifests.py
+++ b/build/images/finalize_manifests.py
@@ -258,11 +258,15 @@
def find_debug_file(filename):
# In the Zircon makefile build, the file to be installed is called
- # foo.strip and the unstripped file is called foo. In the GN build,
- # the file to be installed is called foo and the unstripped file has
- # the same name in the exe.unstripped or lib.unstripped subdirectory.
+ # foo.strip and the unstripped file is called foo. In the new Zircon
+ # GN build, the file to be installed is called foo and the unstripped
+ # file is called foo.debug. In the Fuchsia GN build, the file to be
+ # installed is called foo and the unstripped file has the same name in
+ # the exe.unstripped or lib.unstripped subdirectory.
if filename.endswith('.strip'):
debugfile = filename[:-6]
+ elif os.path.exists(filename + '.debug'):
+ debugfile = filename + '.debug'
else:
dir, file = os.path.split(filename)
if file.endswith('.so') or '.so.' in file:
diff --git a/build/images/guest/BUILD.gn b/build/images/guest/BUILD.gn
index 6812232..c4c5fbc 100644
--- a/build/images/guest/BUILD.gn
+++ b/build/images/guest/BUILD.gn
@@ -53,7 +53,7 @@
testonly = true
bootfs_manifest = boot_manifest
- bootfs_zircon_groups = "misc,test"
+ bootfs_zircon_groups = "all"
args = []
deps = []
@@ -224,10 +224,15 @@
":devmgr_config.txt",
":guest.manifest",
]
- inputs = [
- "${zircon_build_dir}/zircon.zbi",
- boot_manifest,
- ]
+ foreach(image, zircon_images) {
+ if (image.cpu == current_cpu && image.type == "zbi" &&
+ image.name == current_cpu) {
+ inputs = [
+ "$zircon_root_build_dir/${image.path}",
+ ]
+ }
+ }
+ inputs += [ boot_manifest ]
manifest = [
{
outputs = [
diff --git a/build/images/manifest.gni b/build/images/manifest.gni
index e13f2cf..0c2451d 100644
--- a/build/images/manifest.gni
+++ b/build/images/manifest.gni
@@ -15,38 +15,20 @@
root_build_dir),
]
- # Manifest files describing extra libraries from a Zircon build
- # not included in `zircon_boot_manifests`, such as an ASan build.
- # Can be either // source paths or absolute system paths.
- #
- # Since Zircon manifest files are relative to a Zircon source directory
- # rather than to the directory containing the manifest, these are assumed
- # to reside in a build directory that's a direct subdirectory of the
- # Zircon source directory and thus their contents can be taken as
- # relative to `get_path_info(entry, "dir") + "/.."`.
- # TODO(mcgrathr): Make Zircon manifests self-relative too and then
- # merge this and toolchain_manifests into generic aux_manifests.
- if (zircon_use_asan) {
- zircon_aux_manifests = [ "$zircon_build_abi_dir/bootfs.manifest" ]
- } else {
- zircon_aux_manifests = [ "$zircon_asan_build_dir/bootfs.manifest" ]
- }
-
- # Manifest files describing files to go into the `/boot` filesystem.
- # Can be either // source paths or absolute system paths.
- # `zircon_boot_groups` controls which files are actually selected.
- #
- # Since Zircon manifest files are relative to a Zircon source directory
- # rather than to the directory containing the manifest, these are assumed
- # to reside in a build directory that's a direct subdirectory of the
- # Zircon source directory and thus their contents can be taken as
- # relative to `get_path_info(entry, "dir") + "/.."`.
- zircon_boot_manifests = [ "$zircon_build_dir/bootfs.manifest" ]
-
# Extra args to globally apply to the manifest generation script.
extra_manifest_args = []
}
+foreach(image, zircon_images) {
+ if (image.type == "manifest") {
+ if (image.name == "legacy-image-$target_cpu") {
+ zircon_boot_manifests = [ "$zircon_root_build_dir/${image.path}" ]
+ } else if (image.name == "asan-libs" && image.cpu == target_cpu) {
+ zircon_aux_manifests = [ "$zircon_root_build_dir/${image.path}" ]
+ }
+ }
+}
+
# Action target that generates a response file in GN's "shlex" format.
#
# Parameters
@@ -193,7 +175,7 @@
}
sources += zircon_aux_manifests + zircon_boot_manifests
foreach(manifest, zircon_aux_manifests + zircon_boot_manifests) {
- manifest_cwd = get_path_info(rebase_path(manifest), "dir") + "/.."
+ manifest_cwd = rebase_path(zircon_root_build_dir)
response_file_contents += [
"--cwd=$manifest_cwd",
"--manifest=" + rebase_path(manifest),
@@ -220,6 +202,10 @@
response_file_contents += [
"--exclude=bin/devhost",
"--exclude=bin/devhost.asan",
+
+ # Elide the empty file deposited by the Zircon build, since
+ # we will add our own.
+ "--exclude=config/devmgr",
]
}
@@ -243,7 +229,7 @@
response_file_contents += [ "--groups=${manifest.groups}" ]
sources += zircon_boot_manifests
foreach(manifest, zircon_boot_manifests) {
- manifest_cwd = get_path_info(rebase_path(manifest), "dir") + "/.."
+ manifest_cwd = rebase_path(zircon_root_build_dir)
response_file_contents += [
"--cwd=$manifest_cwd",
"--manifest=" + rebase_path(manifest),
diff --git a/build/images/zedboot/BUILD.gn b/build/images/zedboot/BUILD.gn
index 1bb2a68..cc1b270 100644
--- a/build/images/zedboot/BUILD.gn
+++ b/build/images/zedboot/BUILD.gn
@@ -100,7 +100,7 @@
":zedboot.manifest",
]
inputs = [
- "${zircon_build_dir}/kernel.zbi",
+ zircon_kernel_zbi,
manifest_file,
]
manifest = [
diff --git a/build/zircon/create_gn_rules.py b/build/zircon/create_gn_rules.py
index 8c2df11..85a0c1e 100755
--- a/build/zircon/create_gn_rules.py
+++ b/build/zircon/create_gn_rules.py
@@ -92,8 +92,10 @@
return
if current_list and current_map:
raise Exception('Found both map-style and list-style section')
- result[current_section] = (current_map if current_map
- else current_list)
+ if current_map:
+ result[current_section] = current_map
+ elif current_list:
+ result[current_section] = current_list
for line in lines:
section_match = section_exp.match(line)
if section_match:
@@ -118,9 +120,8 @@
# name: foo/bar.h
# path: <SOURCE|BUILD>/somewhere/under/zircon/foo/bar.h
(full_path, changes) = re.subn('^SOURCE', context.source_base, path)
- build_base = context.tool_build_base if is_tool else context.user_build_base
if not changes:
- (full_path, changes) = re.subn('^BUILD', build_base, path)
+ (full_path, changes) = re.subn('^BUILD', context.build_base, path)
if not changes:
raise Exception('Unknown pattern type: %s' % path)
folder = None
@@ -444,12 +445,10 @@
class GenerationContext(object):
'''Describes the context in which GN rules should be generated.'''
- def __init__(self, out_dir, source_base, user_build_base, tool_build_base,
- templates):
+ def __init__(self, out_dir, source_base, build_base, templates):
self.out_dir = out_dir
self.source_base = source_base
- self.user_build_base = user_build_base
- self.tool_build_base = tool_build_base
+ self.build_base = build_base
self.templates = templates
@@ -458,21 +457,15 @@
parser.add_argument('--out',
help='Path to the output directory',
required=True)
- parser.add_argument('--staging',
- help='Path to the staging directory',
+ parser.add_argument('--zircon-build',
+ help='Path to the Zircon build directory',
required=True)
- parser.add_argument('--zircon-user-build',
- help='Path to the Zircon "user" build directory',
- required=True)
- parser.add_argument('--zircon-tool-build',
- help='Path to the Zircon "tools" build directory',
+ parser.add_argument('--zircon-manifest',
+ help='Path to the Zircon export/manifest file',
required=True)
parser.add_argument('--debug',
help='Whether to print out debug information',
action='store_true')
- parser.add_argument('--make',
- help='Path to make binary',
- required=True)
args = parser.parse_args()
out_dir = os.path.abspath(args.out)
@@ -484,29 +477,13 @@
shutil.rmtree(os.path.join(out_dir, 'tool'), True)
debug = args.debug
- # Generate package descriptions through Zircon's build.
- zircon_dir = os.path.abspath(args.staging)
- shutil.rmtree(zircon_dir, True)
- if debug:
- print('Building Zircon in: %s' % zircon_dir)
- make_args = [
- args.make,
- 'packages',
- 'BUILDDIR=%s' % zircon_dir,
- ]
-
- env = {}
- env['PATH'] = os.environ['PATH']
- if not debug:
- env['QUIET'] = '1'
- subprocess.check_call(make_args, cwd=ZIRCON_ROOT, env=env)
- # Parse package definitions.
+ # Parse package definitions from Zircon's build.
packages = []
- with open(os.path.join(zircon_dir, 'export', 'manifest'), 'r') as manifest:
- package_files = map(lambda line: line.strip(), manifest.readlines())
- for file in package_files:
- with open(os.path.join(zircon_dir, 'export', file), 'r') as pkg_file:
- packages.append(parse_package(pkg_file.readlines()))
+ with open(args.zircon_manifest, 'r') as manifest:
+ for file in manifest:
+ file = file.strip()
+ with open(os.path.join(args.zircon_build, 'export', file), 'r') as pkg_file:
+ packages.append(parse_package(pkg_file.readlines()))
if debug:
print('Found %s packages:' % len(packages))
names = sorted(map(lambda p: p['package']['name'], packages))
@@ -517,8 +494,7 @@
context = GenerationContext(
out_dir,
ZIRCON_ROOT,
- os.path.abspath(args.zircon_user_build),
- os.path.abspath(args.zircon_tool_build),
+ os.path.abspath(args.zircon_build),
TemplateLookup(directories=[SCRIPT_DIR]),
)
for package in packages:
@@ -552,10 +528,14 @@
if debug:
print('Processed %s (%s)' % (name, type))
- board_path = os.path.join(zircon_dir, 'export', 'all-boards.list')
- with open(board_path, 'r') as board_file:
- package = parse_package(board_file.readlines())
- generate_board_list(package, context)
+ board_file_lines = []
+ for cpu in ['arm64', 'x64']:
+ board_path = os.path.join(args.zircon_build, 'export',
+ 'boards-%s.list' % cpu)
+ with open(board_path, 'r') as board_file:
+ board_file_lines += [ '[%s]' % cpu ] + board_file.readlines()
+ package = parse_package(board_file_lines)
+ generate_board_list(package, context)
if __name__ == '__main__':
diff --git a/scripts/build-zircon.sh b/scripts/build-zircon.sh
index f18edf4..6c20f37 100755
--- a/scripts/build-zircon.sh
+++ b/scripts/build-zircon.sh
@@ -6,32 +6,41 @@
readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
readonly ROOT_DIR="$(dirname "${SCRIPT_DIR}")"
-JOBS=`getconf _NPROCESSORS_ONLN` || {
- Cannot get number of processors
- exit 1
-}
-
set -eo pipefail; [[ "${TRACE}" ]] && set -x
usage() {
- echo "$0 <options> <extra-make-arguments>"
+ echo "$0 <options> [<gn_build_arg>=<value> ...] [<ninja_arg> ...]"
echo ""
echo "Options:"
echo " -c: Clean before building"
+ echo " -g: Run GN but not Ninja"
+ echo " -G: Run Ninja but not GN (Ninja will re-run GN as needed)"
echo " -v: Level 1 verbosity"
echo " -V: Level 2 verbosity"
echo " -A: Build with ASan"
echo " -H: Build host tools with ASan"
- echo " -n: Just print make recipes to STDOUT."
+ echo " -n: Just print build commands to STDOUT."
echo " -l: Only build tools; do not build zircon."
echo " -j N: Passed along to make (number of parallel jobs)"
echo " -t <target>: Architecture (GN style) to build, instead of all"
echo " -o <outdir>: Directory in which to put the build-zircon directory."
echo ""
- echo "Additional arguments may be passed to make using standard FOO=bar syntax."
- echo "E.g., build-zircon.sh USE_CLANG=true"
+ echo 'Additional arguments containing `=` will be used as GN build arguments.'
+ echo "Other additional arguments will be passed as extra Ninja arguments."
+ echo "e.g., $0 use_goma=true core-tests-x64"
+ echo ""
+ echo 'Note that -A and -H translate into a `variants=...` build argument.'
+ echo "You can't use those switches and also such a build argument."
+ echo "Note that if GN no build arguments (nor -c) are specified and the file"
+ echo "<outdir>/build-zircon/args.gn already exists, it will be reused."
+ echo "For nontrivial configuration changes, edit the args.gn file by hand"
+ echo "either from scratch or after a run of this script with switches;"
+ echo "then run this script with neither build arguments nor -A or -H."
}
+readonly GN="${ROOT_DIR}/zircon/prebuilt/downloads/gn"
+readonly NINJA="${ROOT_DIR}/zircon/prebuilt/downloads/ninja"
+
declare ASAN="false"
declare CLEAN="false"
declare DRY_RUN="false"
@@ -40,11 +49,16 @@
declare OUTDIR="${ROOT_DIR}/out"
declare VERBOSE="0"
declare -a ARCHLIST=(arm64 x64)
+declare JOBS=0
+declare RUN_GN="true"
+declare RUN_NINJA="true"
-while getopts "AcHhlnj:t:p:o:vV" opt; do
+while getopts "AcgGHhlnj:t:p:o:vV" opt; do
case "${opt}" in
A) ASAN="true" ;;
c) CLEAN="true" ;;
+ g) RUN_NINJA="false" ;;
+ G) RUN_GN="false" ;;
H) HOST_ASAN="true" ;;
h) usage ; exit 0 ;;
n) DRY_RUN="true" ;;
@@ -67,51 +81,90 @@
rm -rf -- "${ZIRCON_BUILDROOT}"
fi
-# These variables are picked up by make from the environment.
-case "${VERBOSE}" in
- 1) QUIET=0 ; V=0 ;;
- 2) QUIET=0 ; V=1 ;;
- *) QUIET=1 ; V=0 ;;
-esac
-export QUIET V
+GN_CMD=("$GN" gen "$ZIRCON_BUILDROOT" --root="${ROOT_DIR}/zircon")
+NINJA_CMD=("$NINJA" -C "$ZIRCON_BUILDROOT")
+VARIANTS=()
+GN_ARGS=()
-if [[ "${ASAN}" = "true" ]]; then
- readonly NOT_ASAN=false
+if $ASAN; then
+ VARIANTS+=(asan)
+fi
+if $HOST_ASAN; then
+ VARIANTS+=(host_asan)
+fi
+if [ ${#VARIANTS[@]} -gt 0 ]; then
+ VARIANTS_ARG='variants = ['
+ for variant in "${VARIANTS[@]}"; do
+ VARIANTS_ARG+="\"${variant}\", "
+ done
+ VARIANTS_ARG+=']'
+ GN_ARGS+=("$VARIANTS_ARG")
+fi
+
+if [[ $VERBOSE -lt 1 ]]; then
+ GN_CMD+=(-q)
+ # Ninja doesn't have a --quiet switch.
+fi
+if [[ $VERBOSE -ge 2 ]]; then
+ NINJA_CMD+=(-v)
+fi
+
+if $DRY_RUN; then
+ NINJA_CMD+=(-n)
+fi
+
+if [[ "$JOBS" != 0 ]]; then
+ NINJA_CMD+=(-j "$JOBS")
+fi
+
+if $TOOLS_ONLY; then
+ NINJA_CMD+=(tools)
else
- readonly NOT_ASAN=true
+ NINJA_CMD+=(legacy-host_tests)
+ for arch in "${ARCHLIST[@]}"; do
+ NINJA_CMD+=("manifest-$arch")
+ done
fi
-if [[ "${DRY_RUN}" = "true" ]]; then
- readonly DRY_RUN_ARGS="-Bnwk"
+for arg in "$@"; do
+ # TODO(BLD-325): Special case for argument used by bot recipes.
+ # Remove this after infra transitions.
+ if [[ "$arg" == GOMACC=* ]]; then
+ goma_dir="$(dirname "${arg#GOMACC=}")"
+ GN_ARGS+=(use_goma=true "goma_dir=\"$goma_dir\"")
+ elif [[ "$arg" == *=* ]]; then
+ GN_ARGS+=("$arg")
+ elif $RUN_NINJA; then
+ NINJA_CMD+=("$arg")
+ else
+ echo >&2 "Extra Ninja arguments given when -g says not to run Ninja"
+ exit 2
+ fi
+done
+
+if [[ ${#GN_ARGS[@]} -gt 0 ]]; then
+ if ! $RUN_GN; then
+ echo >&2 "GN build arguments specified when -G says not to run GN"
+ echo >&2 "Consider editting $ZIRCON_BUILDROOT/args.gn by hand instead."
+ exit 2
+ fi
+ GN_CMD+=("--args=${GN_ARGS[*]}")
+elif $RUN_GN && [[ $VERBOSE -gt 0 && -r "$ZIRCON_BUILDROOT/args.gn" ]]; then
+ echo >&2 "Reusing existing $ZIRCON_BUILDROOT/args.gn file."
fi
-make_zircon_common() {
- (test $QUIET -ne 0 || set -x
- exec make ${DRY_RUN_ARGS} --no-print-directory -C "${ROOT_DIR}/zircon" \
- -j ${JOBS} DEBUG_BUILDROOT=../../zircon "$@")
+run_cmd() {
+ if [[ $VERBOSE -gt 0 ]]; then
+ (set -x; "$@")
+ else
+ "$@"
+ fi
}
-# Build host tools.
-make_zircon_common \
- BUILDDIR=${OUTDIR}/build-zircon HOST_USE_ASAN="${HOST_ASAN}" tools "$@"
-
-if [[ "${TOOLS_ONLY}" = "true" ]]; then
+if ! $RUN_GN && ! $RUN_NINJA; then
+ echo >&2 "Doing nothing since -g and -G say not to."
exit 0
fi
-make_zircon_target() {
- make_zircon_common \
- BUILDROOT=${ZIRCON_BUILDROOT} TOOLS=${OUTDIR}/build-zircon/tools "$@"
-}
-
-for ARCH in "${ARCHLIST[@]}"; do
- # Build without ASan for sysroot. If all of userland will be ASan,
- # then this build is only user libraries.
- make_zircon_target PROJECT="${ARCH}" \
- BUILDDIR_SUFFIX= ENABLE_ULIB_ONLY="${ASAN}" "$@"
-
- # Always build at least the libraries with ASan, but never the sysroot.
- make_zircon_target PROJECT="${ARCH}" \
- BUILDDIR_SUFFIX=-asan USE_ASAN=true ENABLE_BUILD_SYSROOT=false \
- ENABLE_ULIB_ONLY="${NOT_ASAN}"
-done
+! $RUN_GN || run_cmd "${GN_CMD[@]}"
+! $RUN_NINJA || run_cmd "${NINJA_CMD[@]}"
diff --git a/scripts/devshell/build b/scripts/devshell/build
index 97ae565..7d0609f 100755
--- a/scripts/devshell/build
+++ b/scripts/devshell/build
@@ -3,63 +3,16 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-### build Fuchsia
+### Run Ninja to build Fuchsia
-## usage: fx build [ninja option,...] [target,...]
+## usage: fx build [ninja arguments ...]
+## Run `fx build -h` to see Ninja argument details.
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"/lib/vars.sh || exit $?
fx-config-read
function main {
- local args=()
- local have_load=false
- local have_jobs=false
- while [[ $# -gt 0 ]]; do
- case "$1" in
- -l) have_load=true ;;
- -j) have_jobs=true ;;
- esac
- args+=("$1")
- shift
- done
-
- if ! $have_load; then
- if [[ "$(uname -s)" == "Darwin" ]]; then
- # Load level on Darwin is quite different from that of Linux, wherein a
- # load level of 1 per CPU is not necessarily a prohibitive load level. An
- # unscientific study of build side effects suggests that cpus*20 is a
- # reaosnable value to prevent catastrophic load (i.e. user can not kill
- # the build, can not lock the screen, etc).
- local cpus="$(fx-cpu-count)"
- args=("-l" $(($cpus * 20)) "${args[@]}")
- fi
- fi
-
- if ! $have_jobs; then
- local concurrency="$(fx-choose-build-concurrency)"
- # macOS in particular has a low default for number of open file descriptors
- # per process, which is prohibitive for higher job counts. Here we raise
- # the number of allowed file descriptors per process if it appears to be
- # low in order to avoid failures due to the limit. See `getrlimit(2)` for
- # more information.
- local min_limit=$((${concurrency} * 2))
- if [[ $(ulimit -n) -lt "${min_limit}" ]]; then
- ulimit -n "${min_limit}"
- fi
- args=("-j" "${concurrency}" "${args[@]}")
- fi
-
-
- # TERM is passed for the pretty ninja UI
- # PATH is passed as some tools are referenced via $PATH due to platform differences.
- # TMPDIR is passed for Goma on macOS. TMPDIR must be set, or unset, not
- # empty. Some Dart build tools have been observed writing into source paths
- # when TMPDIR="" - it is deliberately unquoted and using the ${+} expansion
- # expression).
- fx-try-locked env -i TERM="${TERM}" PATH="${PATH}" \
- ${NINJA_STATUS+"NINJA_STATUS=${NINJA_STATUS}"} \
- ${TMPDIR+"TMPDIR=$TMPDIR"} \
- "${FUCHSIA_DIR}/buildtools/ninja" -C "${FUCHSIA_BUILD_DIR}" "${args[@]}"
+ fx-run-ninja "${FUCHSIA_DIR}/buildtools/ninja" -C "${FUCHSIA_BUILD_DIR}" "$@"
}
main "$@"
diff --git a/scripts/devshell/build-zircon b/scripts/devshell/build-zircon
index 2071072..12f5507 100755
--- a/scripts/devshell/build-zircon
+++ b/scripts/devshell/build-zircon
@@ -3,13 +3,19 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-### build the kernel
+### Run //scripts/build-zircon.sh to build Zircon.
+## Arguments are passed to build-zircon.sh; run with -h for details.
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"/lib/vars.sh || exit $?
fx-config-read
-echo "Building zircon..."
-"${FUCHSIA_DIR}/scripts/build-zircon.sh" \
+# If `fx set` or `fx build-zircon` has already been run, then the explicit
+# `gn gen` run is superfluous and Ninja will do it as needed.
+if [[ -r "${ZIRCON_BUILDROOT}/args.gn" ]]; then
+ set -- -G "$@"
+fi
+
+fx-run-ninja "${FUCHSIA_DIR}/scripts/build-zircon.sh" \
-t "${FUCHSIA_ARCH}" \
-j "$(fx-choose-build-concurrency)" \
- "${FUCHSIA_BUILD_ZIRCON_ARGS[@]}" "$@"
+ "$@"
diff --git a/scripts/devshell/full-build b/scripts/devshell/full-build
index 3fb0c5a..5218479 100755
--- a/scripts/devshell/full-build
+++ b/scripts/devshell/full-build
@@ -4,7 +4,8 @@
# found in the LICENSE file.
### build zircon, then build fuchsia
-## build zircon, then build fuchsia
+## Runs exactly `fx build-zircon` followed by `fx build`.
+## Arguments are ignored. To pass arguments, run each step separately.
set -e
diff --git a/scripts/devshell/lib/vars.sh b/scripts/devshell/lib/vars.sh
index 4c13d10..b5d8628 100644
--- a/scripts/devshell/lib/vars.sh
+++ b/scripts/devshell/lib/vars.sh
@@ -115,22 +115,14 @@
local -r build_dir="$1"
if [[ "$build_dir" == /* ]]; then
local -r args_file="${build_dir}/args.gn"
- local -r zircon_args_file="${build_dir}.zircon-args"
else
local -r args_file="${FUCHSIA_DIR}/${build_dir}/args.gn"
- local -r zircon_args_file="${FUCHSIA_DIR}/${build_dir}.zircon-args"
fi
- local arch zircon_args
- arch="$(fx-config-glean-arch "$args_file")" || return
- if [[ -f "${zircon_args_file}" ]]; then
- zircon_args=$(<"${zircon_args_file}")
- fi
- shift
+ local arch="$(fx-config-glean-arch "$args_file")" || return
echo > "${FUCHSIA_CONFIG}" "\
# Generated by \`fx set\` or \`fx use\`.
FUCHSIA_BUILD_DIR='${build_dir}'
FUCHSIA_ARCH='${arch}'
-FUCHSIA_BUILD_ZIRCON_ARGS=(${zircon_args})
"
}
@@ -333,3 +325,57 @@
function fx-exit-on-failure {
"$@" || exit $?
}
+
+# Massage a ninja command line to add default -j and/or -l switches.
+# Note this takes the ninja command itself as first argument.
+# This can be used both to run ninja directly or to run a wrapper script.
+function fx-run-ninja {
+ local args=()
+ local have_load=false
+ local have_jobs=false
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -l) have_load=true ;;
+ -j) have_jobs=true ;;
+ esac
+ args+=("$1")
+ shift
+ done
+
+ if ! $have_load; then
+ if [[ "$(uname -s)" == "Darwin" ]]; then
+ # Load level on Darwin is quite different from that of Linux, wherein a
+ # load level of 1 per CPU is not necessarily a prohibitive load level. An
+ # unscientific study of build side effects suggests that cpus*20 is a
+ # reaosnable value to prevent catastrophic load (i.e. user can not kill
+ # the build, can not lock the screen, etc).
+ local cpus="$(fx-cpu-count)"
+ args+=("-l" $(($cpus * 20)))
+ fi
+ fi
+
+ if ! $have_jobs; then
+ local concurrency="$(fx-choose-build-concurrency)"
+ # macOS in particular has a low default for number of open file descriptors
+ # per process, which is prohibitive for higher job counts. Here we raise
+ # the number of allowed file descriptors per process if it appears to be
+ # low in order to avoid failures due to the limit. See `getrlimit(2)` for
+ # more information.
+ local min_limit=$((${concurrency} * 2))
+ if [[ $(ulimit -n) -lt "${min_limit}" ]]; then
+ ulimit -n "${min_limit}"
+ fi
+ args+=("-j" "${concurrency}")
+ fi
+
+ # TERM is passed for the pretty ninja UI
+ # PATH is passed as some tools are referenced via $PATH due to platform differences.
+ # TMPDIR is passed for Goma on macOS. TMPDIR must be set, or unset, not
+ # empty. Some Dart build tools have been observed writing into source paths
+ # when TMPDIR="" - it is deliberately unquoted and using the ${+} expansion
+ # expression).
+ fx-try-locked env -i TERM="${TERM}" PATH="${PATH}" \
+ ${NINJA_STATUS+"NINJA_STATUS=${NINJA_STATUS}"} \
+ ${TMPDIR+"TMPDIR=$TMPDIR"} \
+ "${args[@]}"
+}
diff --git a/scripts/devshell/run-host-tests b/scripts/devshell/run-host-tests
index 3ffd338..61653ec 100755
--- a/scripts/devshell/run-host-tests
+++ b/scripts/devshell/run-host-tests
@@ -40,7 +40,7 @@
done
if [[ $ZIRCON -eq 1 ]]; then
- host_test_dir="${ZIRCON_BUILD_DIR}/host_tests"
+ host_test_dir="${ZIRCON_BUILDROOT}/host_tests"
fx-command-run build-zircon "-v"
else
host_test_dir="${FUCHSIA_BUILD_DIR}/host_tests"
diff --git a/scripts/devshell/set b/scripts/devshell/set
index 6b6153d..b4478ab 100755
--- a/scripts/devshell/set
+++ b/scripts/devshell/set
@@ -134,8 +134,8 @@
## build.
## This flag is deprecated, please use one of
## --available, --preinstall, or --monolith.
-## --zircon-arg ARG Additional arguments to pass to Zircon make. Can be given
-## multiple times.
+## --zircon-arg ARG Additional arguments to pass to build-zircon.sh.
+## Can be given multiple times.
##
## Example:
##
@@ -238,7 +238,7 @@
local preinstall=()
local monolith=()
local packages=()
- local zircon_args=()
+ local zircon_args=(-v -g -t "$arch")
local extra_packages=()
local build_dir=
local variant=
@@ -501,6 +501,7 @@
# Add goma or ccache settings as appropriate.
if [[ "${use_goma}" -eq 1 ]]; then
gn_args+=" use_goma=true goma_dir=\"${goma_dir}\""
+ zircon_args+=(use_goma=true "goma_dir=\"${goma_dir}\"")
elif [[ "${ccache}" -eq 1 ]]; then
gn_args+=" use_ccache=true"
fi
@@ -545,14 +546,14 @@
gn_args+=" select_variant=[${variant}]"
fi
- mkdir -p "${build_dir}"
- echo "${zircon_args[@]}" > "${build_dir}.zircon-args"
-
# Using a subshell with -x prints out the gn command precisely with shell
# quoting so a cut&paste to the command line works. Always show the real
# meaning of what this script does so everyone learns how GN works.
(
- set -x
+ set -x -e
+ # Zircon's `gn gen` phase has to be done before Fuchsia's `gn gen`.
+ # The former produces the legacy-$cpu.json file consumed by the latter.
+ "${FUCHSIA_DIR}/scripts/build-zircon.sh" "${zircon_args[@]}"
"${FUCHSIA_DIR}/buildtools/gn" ${gn_cmd} "${build_dir}" \
"${gn_switches[@]}" --args="${gn_args}" "$@"
# If GN failed, don't update .config.
