blob: adce80e3deb069c6f2fbcdf68edccc7724e83e86 [file]
#!/usr/bin/env fuchsia-vendored-python
# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import json
import logging
import os
import sys
from typing import Any, TextIO
from assembly import (
AIBCreator,
FileEntry,
FilePath,
PackagedDriverDetails,
PackageDetails,
PackageManifest,
)
from assembly.assembly_input_bundle import (
AssemblyInputBundleCreationException,
CompiledPackageDefinitionFromGN,
)
from depfile import DepFile
from serialization import instance_from_dict, json_load
logger = logging.getLogger()
BOOTFS_COMPILED_PACKAGE_ALLOWLIST = [
"bootstrap",
"fshost",
"for-test2",
"root",
"toolbox",
] # test package
def create_bundle(args: argparse.Namespace) -> int:
"""Create an Assembly Input Bundle (AIB)."""
aib_creator = AIBCreator(
args.outdir,
args.experimental,
args.testonly,
)
# Set the feature set + build type combinations that we allow the contents
# of this AIB to be included in.
if args.allowed_in:
assert (
"everything" not in args.allowed_in or len(args.allowed_in) == 1
), "if 'everything' is added to allowed_in, it should be the only entry"
for rule in args.allowed_in:
aib_creator.add_allowed_in(rule)
# Set the feature set + build type combinations that we expect the contents
# of this AIB to be included in.
if args.scrutiny_required:
assert (
"everything" not in args.scrutiny_required
or len(args.scrutiny_required) == 1
), "if 'everything' is added to scrutiny_required, it should be the only entry"
for rule in args.scrutiny_required:
aib_creator.add_scrutiny_required(rule)
# Set the feature set + build type combinations that the AIB should
# automatically be included in.
if args.auto_include_in:
assert (
"everything" not in args.auto_include_in
or len(args.auto_include_in) == 1
), "if 'everything' is added to auto_include_in, it should be the only entry"
for rule in args.auto_include_in:
aib_creator.add_auto_include_in(rule)
# Add the base and cache packages, if they exist.
if args.base_pkg_list:
add_pkg_list_from_file(aib_creator, args.base_pkg_list, "base")
if args.cache_pkg_list:
add_pkg_list_from_file(aib_creator, args.cache_pkg_list, "cache")
if args.flexible_pkg_list:
add_pkg_list_from_file(aib_creator, args.flexible_pkg_list, "flexible")
if args.system_pkg_list:
add_pkg_list_from_file(aib_creator, args.system_pkg_list, "system")
if args.bootfs_pkg_list:
add_pkg_list_from_file(aib_creator, args.bootfs_pkg_list, "bootfs")
if args.bootfs_or_base_pkg_list:
add_pkg_list_from_file(
aib_creator, args.bootfs_or_base_pkg_list, "bootfs_or_base"
)
if args.on_demand_pkg_list:
add_pkg_list_from_file(
aib_creator, args.on_demand_pkg_list, "on_demand"
)
if args.anchored_automatic_pkg_list:
add_pkg_list_from_file(
aib_creator, args.anchored_automatic_pkg_list, "anchored_automatic"
)
if args.anchored_on_demand_pkg_list:
add_pkg_list_from_file(
aib_creator, args.anchored_on_demand_pkg_list, "anchored_on_demand"
)
if args.shell_cmds_list:
add_shell_commands_from_file(aib_creator, args.shell_cmds_list)
if args.compiled_packages:
add_compiled_packages_from_file(aib_creator, args.compiled_packages)
if args.drivers_list:
add_driver_list_from_file(
aib_creator,
args.drivers_list,
aib_creator.provided_driver_details,
)
if args.config_data_list:
for config_data_entry_file in args.config_data_list:
with open(config_data_entry_file) as config_data_entry:
add_config_data_entries_from_file(
aib_creator, config_data_entry
)
if args.bootfs_files_package:
aib_creator.bootfs_files_package = args.bootfs_files_package
if args.bootfs_files_list:
for bootfs_files_entry_file in args.bootfs_files_list:
with open(bootfs_files_entry_file) as bootfs_files_entry:
add_bootfs_files_from_list(aib_creator, bootfs_files_entry)
if args.kernel_cmdline:
add_kernel_cmdline_from_file(aib_creator, args.kernel_cmdline)
# Add any bootloaders.
if args.qemu_kernel:
aib_creator.qemu_kernel = args.qemu_kernel
if args.kernel:
aib_creator.kernel.path = args.kernel
if args.memory_buckets:
add_memory_buckets(aib_creator, args.memory_buckets)
# Create the AIB itself.
try:
(assembly_input_bundle, assembly_config, deps) = aib_creator.build()
except AssemblyInputBundleCreationException as e:
print(
f"\n\nERROR: Unable to create AIB\n\n Reason: {e}\n\n",
file=sys.stderr,
)
return 1
# Write out a dep file if one is requested.
if args.depfile:
with open(args.depfile, "w") as depfile:
DepFile.from_deps(assembly_config, deps).write_to(depfile)
# Write out a fini manifest of the files that have been copied, to create a
# package or archive that contains all of the files in the bundle.
if args.export_manifest:
with open(args.export_manifest, "w") as export_manifest:
assembly_input_bundle.write_fini_manifest(
export_manifest, base_dir=args.outdir
)
return 0
def add_pkg_list_from_file(
aib_creator: AIBCreator, pkg_list_file: TextIO, pkg_set_name: str
) -> None:
pkg_list: list[str] = _read_json_file(pkg_list_file) # type: ignore
for package in [PackageDetails(m, pkg_set_name) for m in pkg_list]:
if package in aib_creator.packages:
raise ValueError(f"duplicate pkg manifest found: {package.package}")
aib_creator.packages.add(package)
def add_kernel_cmdline_from_file(
aib_creator: AIBCreator, kernel_cmdline_file: TextIO
) -> None:
cmdline_list: list[str] = _read_json_file(kernel_cmdline_file) # type: ignore
for cmd in cmdline_list:
if cmd in aib_creator.kernel.args:
raise ValueError(f"duplicate kernel cmdline arg found: {cmd}")
aib_creator.kernel.args.add(cmd)
def add_memory_buckets(
aib_creator: AIBCreator, memory_buckets: list[str]
) -> None:
for memory_bucket in memory_buckets:
if memory_bucket in aib_creator.memory_buckets:
raise ValueError(f"duplicate memory bucket found: {memory_bucket}")
aib_creator.memory_buckets.add(memory_bucket)
def add_driver_list_from_file(
aib_creator: AIBCreator,
driver_list_file: TextIO,
driver_list: list[PackagedDriverDetails],
) -> None:
# cross-check the base and bootfs_package sets for the driver before adding
# it to the target driver_list.
driver_details_list = _read_json_file(driver_list_file)
for driver_details in driver_details_list:
if driver_details["package_target"] in aib_creator.packages:
raise ValueError(
f"duplicate pkg manifest found: {driver_details['package_target']}"
)
driver_list.append(
PackagedDriverDetails(
driver_details["package_target"],
driver_details["set"],
driver_details["driver_components"],
)
)
def add_shell_commands_from_file(
aib_creator: AIBCreator, shell_commands_list_file: TextIO
) -> None:
"""
[
{
"components": [
"ls"
],
"package": "ls"
}
]
"""
loaded_file = _read_json_file(shell_commands_list_file)
for command in loaded_file:
package = command["package"]
components = command["components"]
bootfs_package = command.get("bootfs_package", False)
if bootfs_package:
aib_creator.bootfs_shell_commands[package].extend(
["bin/" + component for component in components]
)
else:
aib_creator.shell_commands[package].extend(
["bin/" + component for component in components]
)
def add_config_data_entries_from_file(
aib_creator: AIBCreator, config_data_entries: TextIO
) -> None:
"""
config_data_entries schema:
[
{
'package_name': 'example_package',
'destination': 'foo.txt',
'source': 'src/sys/example/configs/example.json'
}
]
"""
_config_data = _read_json_file(config_data_entries)
for definition in _config_data:
entry = FileEntry(
definition["source"],
f"meta/data/{definition['package_name']}/{definition['destination']}",
)
aib_creator.config_data.append(entry)
def add_compiled_packages_from_file(
aib_creator: AIBCreator, compiled_packages: TextIO
) -> None:
"""
compiled_packages should be
list[CompiledPackageDefinition]
"""
_compiled_packages: list[dict[str, Any]] = _read_json_file(
compiled_packages
)
for package_dict in _compiled_packages:
# Parse the dict into an object.
package_def = instance_from_dict(
CompiledPackageDefinitionFromGN, package_dict
)
# If a bootfs package, validate the package name against the allowlist
# for compiled packages in bootfs.
if (
package_def.bootfs_package
and package_def.name not in BOOTFS_COMPILED_PACKAGE_ALLOWLIST
):
raise ValueError(
f"Compiled package {package_def.name} not in bootfs allowlist!"
)
aib_creator.compiled_packages.append(package_def)
def add_bootfs_files_from_list(
aib_creator: AIBCreator, bootfs_files: TextIO
) -> None:
"""
bootfs_files schema:
[
{
'destination': 'bin/bar',
'source': 'src/sys/example/configs/example.json'
}
]
"""
_bootfs_files = _read_json_file(bootfs_files)
for entry in _bootfs_files:
# Not all distribution manifests have the source and destination pairs.
# For an example see: dart_kernel.gni
if "source" in entry and "destination" in entry:
aib_creator.bootfs_files.add(
FileEntry(entry["source"], entry["destination"])
)
def _read_json_file(pkg_list_file: TextIO) -> list[dict[str, Any]]:
try:
return json.load(pkg_list_file)
except:
logger.exception(f"While parsing {pkg_list_file.name}")
raise
def find_blob_in_manifests(
blob_to_find: str, bundle_dir: str, manifests_to_search: list[FilePath]
) -> list[tuple[FilePath, FilePath]]:
found_at: list[tuple[FilePath, FilePath]] = []
known_manifests = set(manifests_to_search)
i = 0
while i < len(manifests_to_search):
pkg_manifest_path = manifests_to_search[i]
i += 1
with open(
os.path.join(bundle_dir, pkg_manifest_path), "r"
) as pkg_manifest_file:
manifest = json_load(PackageManifest, pkg_manifest_file)
if not manifest.blob_sources_relative:
raise ValueError(
f"Unexpected non-relative paths in AIB package manifest: {pkg_manifest_path}"
)
for blob in manifest.blobs:
if blob.merkle == blob_to_find:
found_at.append((pkg_manifest_path, blob.path))
for subpackage in manifest.subpackages:
subpackage_manifest_path = os.path.join(
os.path.dirname(pkg_manifest_path), subpackage.manifest_path
)
# remove `<dir>/../` sequences if present
subpackage_manifest_path = os.path.relpath(
subpackage_manifest_path
)
if subpackage_manifest_path not in known_manifests:
manifests_to_search.append(subpackage_manifest_path)
known_manifests.add(subpackage_manifest_path)
return found_at
def main() -> int:
parser = argparse.ArgumentParser(
description="Tool for creating Assembly Input Bundles in-tree, for use with out-of-tree assembly"
)
sub_parsers = parser.add_subparsers(
title="Commands",
description="Commands for working with Assembly Input Bundles",
)
###
#
# 'assembly_input_bundle_tool create' subcommand parser
#
bundle_creation_parser = sub_parsers.add_parser(
"create", help="Create an Assembly Input Bundle"
)
bundle_creation_parser.add_argument(
"--outdir",
required=True,
help="Path to the outdir that will contain the AIB",
)
bundle_creation_parser.add_argument(
"--testonly",
action="store_true",
help="If an AIB is testonly then it can never be allowed on user builds",
)
bundle_creation_parser.add_argument(
"--allowed-in",
type=str,
action="append",
)
bundle_creation_parser.add_argument(
"--scrutiny-required",
type=str,
action="append",
)
bundle_creation_parser.add_argument(
"--auto-include-in",
type=str,
action="append",
)
bundle_creation_parser.add_argument(
"--experimental",
action="store_true",
)
bundle_creation_parser.add_argument(
"--base-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'base' package set",
)
bundle_creation_parser.add_argument(
"--bootfs-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'bootfs' package set",
)
bundle_creation_parser.add_argument(
"--bootfs-or-base-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'bootfs-or-base' package set",
)
bundle_creation_parser.add_argument(
"--on-demand-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'on-demand' package set",
)
bundle_creation_parser.add_argument(
"--drivers-list",
type=argparse.FileType("r"),
help="Path to a json list of driver details",
)
bundle_creation_parser.add_argument(
"--shell-cmds-list",
type=argparse.FileType("r"),
help="Path to a json list of dictionaries with the manifest path as key and a list of shell_command components as the value",
)
bundle_creation_parser.add_argument(
"--cache-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'cache' package set",
)
bundle_creation_parser.add_argument(
"--flexible-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'flexible' package set",
)
bundle_creation_parser.add_argument(
"--system-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'system' package set",
)
bundle_creation_parser.add_argument(
"--anchored-automatic-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'anchored automatic' package set",
)
bundle_creation_parser.add_argument(
"--anchored-on-demand-pkg-list",
type=argparse.FileType("r"),
help="Path to a json list of package manifests for the 'anchored on-demand' package set",
)
bundle_creation_parser.add_argument(
"--kernel-cmdline",
type=argparse.FileType("r"),
help="Path to a json list of kernel cmdline arguments",
)
bundle_creation_parser.add_argument(
"--qemu-kernel", help="Path to the qemu kernel"
)
bundle_creation_parser.add_argument(
"--kernel", help="Path to the Zircon kernel ZBI"
)
bundle_creation_parser.add_argument(
"--depfile",
help="Path to write a dependency file to",
)
bundle_creation_parser.add_argument(
"--export-manifest",
help="Path to write a FINI manifest of the contents of the AIB",
)
bundle_creation_parser.add_argument(
"--config-data-list",
action="append",
help="Path to a json file of config-data entries, may be specified multiple times",
)
bundle_creation_parser.add_argument(
"--bootfs-files-package",
help="Path to a package manifest that points to files to include in bootfs",
)
bundle_creation_parser.add_argument(
"--bootfs-files-list",
action="append",
help="Path to a json file of bootfs-file entries, may be specified multiple times",
)
bundle_creation_parser.add_argument(
"--compiled-packages",
type=argparse.FileType("r"),
help="Path to a json file of compiled package configuration",
)
bundle_creation_parser.add_argument(
"--memory-buckets",
action="append",
help="Path to a json file of memory buckets",
)
bundle_creation_parser.set_defaults(handler=create_bundle)
args: argparse.Namespace = parser.parse_args()
if "handler" in args:
# Dispatch to the handler fn.
return args.handler(args)
else:
# argparse doesn't seem to automatically catch that not subparser was
# called, and so if there isn't a handler function (which is set by
# having specified a subcommand), then just display usage instead of
# a cryptic KeyError.
parser.print_help()
return 0
if __name__ == "__main__":
sys.exit(main())