build/assembly/assembly_input_bundle.gni

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import("//build/assembly/assembly_input_bundle_archive.gni")
import("//build/assembly/assembly_input_bundle_package.gni")
import("//build/bazel/bazel_inputs.gni")
import("//build/components/fuchsia_package.gni")
import("//build/group_with_inputs.gni")
import("//build/python/python_action.gni")

# Creates an Assembly Input Bundle from sets of deps.
#
# NOTE: This is not yet able to support all categories of inputs that are in an
#       AIB.  That support will be added in future CLs.  See the parameters
#       below for the categories that are currently supported.
#
# NOTE: This template DOES NOT use GN metadata, all labels for packages must be
# the actual target that creates the package.
#
# Parameters
#
#  bundle_name (optional; default: target_name)
#    [string] A different name for the bundle, if not the name of the target.
#
#  bundle_dir (optional; default: target_out_dir)
#    [path] A different directory to write the bundle into, if not to write it
#    to '$target_out_dir/$bundle_name'.
#
#  base_packages [optional]
#    [list, GN labels] A list of GN labels of fuchsia_package targets to
#    include in the base package set.
#
#    Note: These are direct labels for specific targets, not deps to walk for
#    metadata.  If the target isn't a package target, it will cause an error
#    like:
#      "no dependency provides the input <package_name>/package_manifest.json"
#
#   base_driver_packages (optional)
#     [list, GN scopes] A list of GN scopes that hold the driver packages to
#     include in the base package set. Packages listed here should not be
#     listed in the base_packages and will be included automatically in
#     the base package set as driver packages. The scope must have a
#     `package_target` field pointing to the GN target of the fuchsia_package,
#     and `driver_components` field containing a list of relative paths to
#     driver components provided by the package, e.g. "meta/driver.cm"
#
#  cache_packages [optional]
#    [list, GN labels] A list of GN labels of fuchsia_package targets to
#    include in the cache package set.
#
#  flexible_packages [optional]
#    [list, GN labels] A list of GN labels of fuchsia_package targets that
#    assembly may choose to put in base, cache, or elsewhere depending on the
#    assembly context.
#
#  bootfs_packages [optional]
#    [list, GN labels] A list of GN labels of fuchsia_package targets to
#    include in the bootfs package set.
#
#    Note: These are direct labels for specific targets, not deps to walk for
#    metadata.  If the target isn't a package target, it will cause an error
#    like:
#      "no dependency provides the input <package_name>/package_manifest.json"
#
#   boot_driver_packages [optional]
#     [list, GN scopes] A list of GN scopes that hold the driver packages to
#     include in the bootfs package set. Packages listed here should not be
#     listed in the bootfs_packages and will be included automatically in
#     the bootfs package set as driver packages. The scope must have a
#     `package_target` field pointing to the GN target of the fuchsia_package,
#     and `driver_components` field containing a list of relative paths to
#     driver components provided by the package, e.g. "meta/driver.cm"
#
#  bootfs_files_labels [optional]
#    [list, GN labels] A list of GN labels of bootfs_files_for_assembly()
#    targets to include in bootfs_files provided by this AIB.
#
#    Note: These are direct labels for specific targets, not deps to walk for
#    metadata. If the target isn't a bootfs_files_for_assembly target, it will
#    cause an error like:
#      "no dependency provides the input <package_name>/bootfs_files.json"
#
#  config_data_labels [optional]
#    [list, GN labels] A list of GN labels of config_data_for_assembly() targets
#    to include in the config_data provided by this AIB.
#
#    Note: These are direct labels for specific targets, not deps to walk for
#    metadata.  If the target isn't a config_data_for_assembly() target, it will
#    cause an error like:
#      "no dependency provides the input <package_name>/config_data_entry.json"
#
#  config_data [optional]
#    [list, scopes] A list of scopes, defining each config_data associated with
#    a given package, and the source, label, and destination required to add it
#    in bootfs using assembly.
#
#    Example w/ source-files:
#    config_data = [
#      {
#        package_name = "example_package"
#        files = [
#          {
#            source =
#                "//src/path_to_config/config1.json"
#            destination = "config1.json"
#          },
#          {
#            source = "//src/path_to_config/other_stuff.txt"
#            destination = "other_stuff.txt"
#          },
#        ]
#      },
#    ]
#
#    Example w/ build-output:
#    config_data = [
#      {
#        package_name = "example_package"
#        label = "//src/package/path:example_package"
#        files = [
#          {
#            source =
#                "//src/path_to_config/config1.json"
#            destination = "config1.json"
#          },
#          {
#            source = "//src/path_to_config/other_stuff.txt"
#            destination = "other_stuff.txt"
#          },
#        ]
#      },
#    ]
#
#  qemu_kernel (optional; default: false)
#    [path] Path to the qemu kernel.
#
#  kernel_cmdline (optional: default: [])
#    [list of strings] Kernel cmdline arguments.
#
#  shell_commands (optional; default: empty)
#    [list of scopes] A list of scopes that describe the shell commands for each
#    listed package
#
#    Example:
#    shell_commands = [
#      {
#        package = "//third_party/sbase"
#        components = [ "ls" ]
#      },
#    ]
#
#  compiled_packages [optional]
#     [list of GN scopes] List of GN scopes of `CompiledPackageDefinition`s
#     that describe packages that are to be built dynamically by Assembly, for
#     example, the `core` package. This is passed directly through to the AIB
#     config so all paths should be rebased by the caller.
#
#     Example:
#
#     # Add a core shard
#     compiled_packages = [
#       {
#         name = "core"
#         component_shards = [
#           {
#             component_name = "core"
#             shards = rebase_path(
#                     "//src/sys/process-resolver/meta/process_resolver.core_shard.cml",
#                     root_build_dir)
# .         },
# .         ...
#         ]
#       },
#     ]
#
#  component_includes [optional]
#     [list of FileEntry] List of source/destination pairs related to a compiled
#     package in the compiled_packages list.
#     that specifies cml files that can be included by any cml shards in any
#     platform AIB for the given package. These files will be included
#     in the Assembly Input Bundle.
#
#  create_aib_package (optional; default: true)
#    [boolean] Set to true to also create a package that contains the contents
#    of the AIB.
#      target: '{$target_name}.pkg'
#      outputs: [
#                 '${bundle_dir}/${bundle_name}.pkg/meta.far',
#                 '${bundle_dir}/${bundle_name}.pkg/package_manifest.json',
#               ]
#
#  create_aib_archive (optional; default: true)
#    [boolean] Set to true to create a tgz archive that contains the contents of
#    the AIB.
#      target: '{$target_name}.tgz'
#      outputs: [ '${bundle_dir}/${bundle_name}.tgz' ]
#
#
#  Outputs
#    A directory structure and manifest that matches that documented in
#    //build/python/modules/assembly/assembly_input_bundle.py.
#
#   manifest path:
#   $target_out_dir/$target_name/assembly_config.json
#
#
# GN usual meanings
#  testonly, visibility
template("assembly_input_bundle") {
  bundles_dir = target_out_dir
  if (defined(invoker.bundles_dir)) {
    bundles_dir = invoker.bundles_dir
  }

  bundle_name = target_name
  if (defined(invoker.bundle_name)) {
    bundle_name = invoker.bundle_name
  }

  assert(bundle_name != "resources",
         "The bundle name 'resources' is reserved for internal use")

  labels = {
    # The AIB itself
    assembly_input_bundle = "$target_name.bundle"

    # The assembly bundle package and archive labels
    assembly_input_bundle_package = "${target_name}.pkg"
    assembly_input_bundle_archive = "${target_name}.tgz"

    bazel_inputs = "${target_name}_bazel_inputs"

    bootfs_files_package = "${target_name}.bootfs_files_package"
    config_data = "${target_name}.config_data_list"
    config_data_with_verification = "${target_name}.group_with.config_data"
    kernel_cmdline = "${target_name}.kernel_cmdline"
    shell_commands_list = "${target_name}.shell_commands.list"
    compiled_packages = "${target_name}.compiled_packages"
  }

  files = {
    # The directory where all the bundle contents are written to
    assembly_input_bundle_dir = "${bundles_dir}/${bundle_name}"

    # The unpackaged bootfs files that we throw into a package to add to the AIB.
    bootfs_files_package = "$target_out_dir/$target_name.bootfs_files_package/package_manifest.json"

    # The "official" outputs file that we create in that directory
    assembly_input_bundle_config =
        "${assembly_input_bundle_dir}/assembly_config.json"

    # The files that we create as book-keeping between our tasks.
    assembly_input_bundle_depfile = "${assembly_input_bundle_dir}.d"

    # The manifest of all files in the AIB, used to create pkgs and archives.
    assembly_input_bundle_manifest =
        "${assembly_input_bundle_dir}.fini_manifest"

    # The AIB package's meta.far (optionally used)
    assembly_input_bundle_package_metafar =
        "${assembly_input_bundle_dir}.pkg/meta.far"

    # The AIB archive and the manifest used to create it (optionally used)
    assembly_input_bundle_archive = "${assembly_input_bundle_dir}.tgz"
    assembly_input_bundle_archive_manifest =
        "${assembly_input_bundle_dir}.tgz.fini_manifest"

    _gen_files = "${target_gen_dir}/${target_name}"
    shell_commands_list = "${_gen_files}/shell_commands.list"

    # The output path for the generated file which collects config_data from
    # the invoking scope
    config_data = "${assembly_input_bundle_dir}.config_data_list"

    # The output path for the generated file which collects kernel cmdline
    # arguments.
    kernel_cmdline = "${assembly_input_bundle_dir}.kernel_cmdline"

    # The list of user-provided compiled_package_definitions to include in
    # the AIB config
    compiled_packages = "${assembly_input_bundle_dir}.compiled_packages"
  }

  # See if the fuchsia-pkg and archive for the AIB are to be created.
  forward_variables_from(invoker,
                         [
                           "create_aib_package",
                           "create_aib_archive",
                         ])
  if (!defined(create_aib_package)) {
    create_aib_package = true
  }
  if (!defined(create_aib_archive)) {
    create_aib_archive = true
  }

  creation_args = []
  creation_inputs = []
  creation_deps = []
  if (defined(invoker.deps)) {
    creation_deps += invoker.deps
  }

  fuchsia_package(labels.bootfs_files_package) {
    forward_variables_from(invoker, [ "testonly" ])
    if (defined(invoker.bootfs_files_labels)) {
      deps = invoker.bootfs_files_labels
    }
  }

  # Pass the bootfs files package to the assembly tool.
  creation_args += [
    "--bootfs-files-package",
    rebase_path(files.bootfs_files_package, root_build_dir),
  ]
  creation_deps += [ ":${labels.bootfs_files_package}" ]

  # Convert the list of config_data_labels into a set of config_data_entry.json
  # files that are inputs that define config data entry scopes.
  if (defined(invoker.config_data_labels)) {
    _config_data_entries = []

    foreach(entry_label, invoker.config_data_labels) {
      # Assume this is a GN-label to a config_data_for_assembly()
      # instantiation, which is similar to packages.
      _entry_out_dir = get_label_info(entry_label, "target_out_dir")
      _entry_name = get_label_info(entry_label, "name")
      _entry_file = "${_entry_out_dir}/${_entry_name}/config_data_entry.json"
      _config_data_entries += [ _entry_file ]

      creation_args += [
        "--config-data-list",
        rebase_path(_entry_file, root_build_dir),
      ]
    }

    creation_deps += invoker.config_data_labels
    creation_inputs += _config_data_entries
  }

  # Splits config_data inputs from the invoker into:
  # 1) group_with_inputs deps / sources and
  # 2) generated_file contents
  if (defined(invoker.config_data)) {
    config_data_contents = []
    _config_data_deps = []
    _config_data_sources = []
    foreach(definition, invoker.config_data) {
      assert(defined(definition.package_name),
             "config_data definitions must include package_name")
      assert(defined(definition.files),
             "config_data definitions must include source")

      if (defined(definition.label)) {
        _config_data_deps += [ definition.label ]
      }

      foreach(file, definition.files) {
        assert(defined(file.source), "file must have source in scope")
        assert(defined(file.destination), "file must have destination in scope")
        _config_data_sources += [ file.source ]
        config_data_contents += [
          {
            package_name = definition.package_name
            source = rebase_path(file.source, root_build_dir)
            destination = file.destination
          },
        ]
      }
    }
    group_with_inputs(labels.config_data_with_verification) {
      deps = _config_data_deps
      sources = _config_data_sources
    }

    generated_file(labels.config_data) {
      forward_variables_from(invoker, [ "testonly" ])
      contents = config_data_contents
      outputs = [ "${files.config_data}" ]
      output_conversion = "json"
    }

    creation_deps += [
      ":${labels.config_data_with_verification}",
      ":${labels.config_data}",
    ]

    creation_args += [
      "--config-data-list",
      rebase_path(files.config_data, root_build_dir),
    ]
    creation_inputs += [ files.config_data ]
  }

  # Convert each set of package targets into:
  #  1) a set of manifests paths in a file
  #  2) add that file as an input and argument to the AIB creation action
  #  3) add the target labels as dependencies of the AIB creation action

  # This defines a set of package sets that can be looped over by name to find
  # process.  This use of defined() and the invoker is to tolerate undefined,
  # optional, parameters to the template, as `invoker[name]` is not supported
  # with defined
  package_sets = {
    base = []
    if (defined(invoker.base_packages)) {
      foreach(_target, invoker.base_packages) {
        base += [
          {
            package_target = _target
          },
        ]
      }
    }
    cache = []
    if (defined(invoker.cache_packages)) {
      foreach(_target, invoker.cache_packages) {
        cache += [
          {
            package_target = _target
          },
        ]
      }
    }
    flexible = []
    if (defined(invoker.flexible_packages)) {
      foreach(_target, invoker.flexible_packages) {
        flexible += [
          {
            package_target = _target
          },
        ]
      }
    }
    system = []
    if (defined(invoker.system_packages)) {
      foreach(_target, invoker.system_packages) {
        system += [
          {
            package_target = _target
          },
        ]
      }
    }
    bootfs = []
    if (defined(invoker.bootfs_packages)) {
      foreach(_target, invoker.bootfs_packages) {
        bootfs += [
          {
            package_target = _target
          },
        ]
      }
    }
    on_demand = []
    if (defined(invoker.on_demand)) {
      foreach(_target, invoker.on_demand) {
        on_demand += [
          {
            package_target = _target
          },
        ]
      }
    }

    base_drivers = []
    if (defined(invoker.base_driver_packages)) {
      base_drivers = invoker.base_driver_packages
    }

    boot_drivers = []
    if (defined(invoker.boot_driver_packages)) {
      boot_drivers = invoker.boot_driver_packages
    }
  }

  # Loop over the above sets of packages
  foreach(set_name,
          [
            "base",
            "cache",
            "flexible",
            "system",
            "base-drivers",
            "bootfs",
            "boot-drivers",
            "on-demand",
          ]) {
    _package_target_details = []
    _package_target_details = package_sets[string_replace(set_name, "-", "_")]
    _package_targets = []
    _manifest_entries = []

    # Skip empty package sets
    if (_package_target_details != []) {
      # 1a) Calculate the manifest path for each package target
      foreach(details, _package_target_details) {
        assert(
            defined(details.package_target),
            "A target must be defined for each package. (this should be unreachable)")
        _package_targets += [ details.package_target ]
        _package_out_dir =
            get_label_info(details.package_target, "target_out_dir")
        _package_name = get_label_info(details.package_target, "name")
        _manifest_path =
            "${_package_out_dir}/${_package_name}/package_manifest.json"
        _manifest_path_rebased = rebase_path(_manifest_path, root_build_dir)

        if (set_name == "base-drivers" || set_name == "boot-drivers") {
          assert(defined(details.driver_components),
                 "driver_components must be specified for driver targets")
          _manifest_entries += [
            {
              package_target = _manifest_path_rebased
              driver_components = details.driver_components
            },
          ]
        } else {
          _manifest_entries += [ _manifest_path_rebased ]
        }
      }

      _manifest_list = "${target_name}.${set_name}_packages.list"
      _manifest_list_file = "${target_gen_dir}/${_manifest_list}"

      # 1b) Generate a file containing that list
      # For base packages this will be a list of package manifests.
      # For drivers, this will be a list of DriverDetails, containing
      # the package manifests and component file paths.
      generated_file(_manifest_list) {
        forward_variables_from(invoker, [ "testonly" ])
        deps = _package_targets
        outputs = [ "${_manifest_list_file}" ]
        output_conversion = "json"
        contents = _manifest_entries
      }

      # 2) add the file as an input and argument to the AIB creation action
      creation_args += [
        "--${set_name}-pkg-list",
        rebase_path(_manifest_list_file, root_build_dir),
      ]
      creation_inputs += [ _manifest_list_file ]

      # 3) add the file as a dependency of the AIB creation action (linking the
      #    actual package deps transitively, to preserve the route by which they
      #    are added to the AIB:
      #      AIB
      #       +-> AIB.base_packages_list
      #            +-> [ base package deps ]
      #       +-> AIB.cache_packages_list
      #            +-> [ cache package deps ]
      creation_deps += [ ":${_manifest_list}" ]
    }
  }

  # Only add the shell command packages list if it's non-empty
  if (defined(invoker.shell_commands) && invoker.shell_commands != []) {
    shell_commands_json_contents = []

    foreach(shell_command, invoker.shell_commands) {
      assert(
          defined(shell_command.package),
          "shell_command entries must specify a package name: ${shell_command}")
      assert(
          defined(shell_command.components),
          "shell_command components must be a list of strings pointing to binaries that are components in the package that make up the package: ${shell_command}")
      _package_name = get_label_info(shell_command.package, "name")
      shell_commands_json_contents += [
        {
          package = _package_name
          components = shell_command.components
        },
      ]
    }

    generated_file(labels.shell_commands_list) {
      forward_variables_from(invoker,
                             [
                               "testonly",
                               "visibility",
                             ])
      outputs = [ files.shell_commands_list ]
      output_conversion = "json"
      contents = shell_commands_json_contents
    }

    creation_args += [
      "--shell-cmds-list",
      rebase_path(files.shell_commands_list, root_build_dir),
    ]
    creation_inputs += [ files.shell_commands_list ]
    creation_deps += [ ":${labels.shell_commands_list}" ]
  }

  if (defined(invoker.qemu_kernel)) {
    creation_args += [
      "--qemu-kernel",
      rebase_path(invoker.qemu_kernel, root_build_dir),
    ]
  }

  if (defined(invoker.kernel_cmdline)) {
    generated_file(labels.kernel_cmdline) {
      forward_variables_from(invoker, [ "testonly" ])
      outputs = [ files.kernel_cmdline ]
      output_conversion = "json"
      contents = invoker.kernel_cmdline
    }

    creation_args += [
      "--kernel-cmdline",
      rebase_path(files.kernel_cmdline, root_build_dir),
    ]
    creation_inputs += [ files.kernel_cmdline ]
    creation_deps += [ ":${labels.kernel_cmdline}" ]
  }

  if (defined(invoker.compiled_packages)) {
    _compiled_packages = []
    foreach(package, invoker.compiled_packages) {
      _package = {
      }
      _package = {
        forward_variables_from(package, "*", [ "contents" ])
      }
      _contents = []
      if (defined(package.contents)) {
        _inputs = []
        _deps = []
        foreach(entry, package.contents) {
          _entry = {
          }
          _entry = {
            forward_variables_from(entry,
                                   [
                                     "label",
                                     "destination",
                                   ])
          }
          _entry.source = rebase_path("${entry.source}", root_build_dir)
          _contents += [ _entry ]

          _inputs += [ entry.source ]
          _deps += [ entry.label ]
        }
        group_name = "${target_name}.compiled_package.${package.name}.inputs"
        group_with_inputs(group_name) {
          deps = _deps
          inputs = _inputs
        }
        creation_deps += [ ":${group_name}" ]
      }
      _package.contents = _contents
      _compiled_packages += [ _package ]
    }
    generated_file(labels.compiled_packages) {
      forward_variables_from(invoker, [ "testonly" ])
      outputs = [ "${files.compiled_packages}" ]
      output_conversion = "json"
      contents = _compiled_packages
    }

    creation_args += [
      "--compiled-packages",
      rebase_path(files.compiled_packages, root_build_dir),
    ]

    creation_inputs += [ files.compiled_packages ]
    creation_deps += [ ":${labels.compiled_packages}" ]
  }

  # Create the out-of-tree-usable Assembly Input Bundle
  python_action(labels.assembly_input_bundle) {
    forward_variables_from(invoker,
                           [
                             "testonly",
                             "visibility",
                           ])

    binary_label = "//build/assembly/scripts:assembly_input_bundle_tool"

    # The contents of these folders is dynamic, and managed entirely by this
    # action.  Further, this action will need to delete items from these
    # directories that are not added back (on an incremental build, if an item
    # is removed from one of these sets)
    #
    # These folders would grow in size forever, if it was not cleaned out on
    # each incremental build.
    hermetic_action_ignored_prefixes = [
      "${files.assembly_input_bundle_dir}/blobs",
      "${files.assembly_input_bundle_dir}/bootfs",
      "${files.assembly_input_bundle_dir}/compiled_packages",
      "${files.assembly_input_bundle_dir}/config_data",
      "${files.assembly_input_bundle_dir}/kernel",
      "${files.assembly_input_bundle_dir}/packages",
      "${files.assembly_input_bundle_dir}/subpackages",
    ]

    outputs = [ files.assembly_input_bundle_config ]
    depfile = files.assembly_input_bundle_depfile

    args = [
      "create",
      "--outdir",
      rebase_path(files.assembly_input_bundle_dir, root_build_dir),
      "--depfile",
      rebase_path(files.assembly_input_bundle_depfile, root_build_dir),
    ]
    args += creation_args

    # If packaging or archiving the AIB, write out the fini manifest needed to
    # do so.
    if (create_aib_package || create_aib_archive) {
      args += [
        "--export-manifest",
        rebase_path(files.assembly_input_bundle_manifest, root_build_dir),
      ]
      outputs += [ files.assembly_input_bundle_manifest ]
    }

    inputs = creation_inputs
    deps = creation_deps

    metadata = {
      # We insert these barriers to prevent the dependencies of the input bundle
      # from leaking into images "higher up" in the dependency chain.
      package_barrier = []
      assembly_package_barrier = []
      config_package_barrier = []
      distribution_entries_barrier = []
      driver_package_component_files_barrier = []
      system_image_package_barrier = []
      shell_commands_barrier = []
      assembly_input_archives_barrier = []
    }
  }

  # Optionally create the fuchsia-pkg for the AIB.
  if (create_aib_package) {
    assembly_input_bundle_package(labels.assembly_input_bundle_package) {
      forward_variables_from(invoker,
                             [
                               "testonly",
                               "visibility",
                             ])
      package_name = bundle_name
      package_outdir = "${bundles_dir}/${bundle_name}.pkg"
      manifest = files.assembly_input_bundle_manifest
      deps = [ ":${labels.assembly_input_bundle}" ]
    }
  }

  # Optionally create the archive for the AIB.
  if (create_aib_archive) {
    assembly_input_bundle_archive(labels.assembly_input_bundle_archive) {
      forward_variables_from(invoker,
                             [
                               "testonly",
                               "visibility",
                             ])
      archive_name = bundle_name
      archive_outdir = bundles_dir
      manifest = files.assembly_input_bundle_manifest
      deps = [ ":${labels.assembly_input_bundle}" ]

      # If the package was created, include it in the archive.
      if (create_aib_package) {
        meta_far = files.assembly_input_bundle_package_metafar
        deps += [ ":${labels.assembly_input_bundle_package}" ]
      }
    }
  }

  # Make generated AIBs available to Bazel builds.
  bazel_input_resource_directory(labels.bazel_inputs) {
    forward_variables_from(invoker,
                           [
                             "testonly",
                             "visibility",
                           ])
    source_dir = files.assembly_input_bundle_dir
    dest_dir = rebase_path(files.assembly_input_bundle_dir, root_out_dir)
    deps = [ ":${labels.assembly_input_bundle}" ]
  }

  group(target_name) {
    forward_variables_from(invoker,
                           [
                             "testonly",
                             "visibility",
                           ])
    deps = [ ":${labels.bazel_inputs}" ]
    public_deps = [ ":${labels.assembly_input_bundle}" ]
    if (create_aib_package) {
      public_deps += [ ":${labels.assembly_input_bundle_package}" ]
    }
    if (create_aib_archive) {
      public_deps += [ ":${labels.assembly_input_bundle_archive}" ]
    }
  }
}