build/images/assembly/BUILD.gn

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This is a re-implementation of //build/images/BUILD.gn, but using the ffx
# assembly plugin as the main implementation.  Most of the work in this file
# is creating the translation between the Fuchsia Build and the inputs that the
# ffx assembly plugin expects.

import("//build/compiled_action.gni")
import("//build/components/fuchsia_system_package.gni")
import("//build/dist/resource.gni")
import("//build/info/info.gni")
import("//build/packages/package_metadata.gni")
import("//build/security.gni")
import("//build/testing/verify_files_match.gni")
import("//src/developer/ffx/build/ffx_action.gni")

# These imports to be replaced by board-config json files
import("//build/images/filesystem_limits.gni")
import("//build/images/fvm.gni")
import("//build/zbi/zbi.gni")

####
#
# Deal with "bringup" and other zbi-only builds.
#
# These builds set `bootfs_only` to true, and do not include a base/system_image
# package, pkgfs, blobfs, fvm, etc.
#
# This declares an inverted variable that has easier reading and is less likely
# to miss the `!` operator:
#
uses_base_package = !bootfs_only

####
#
# All the stuff in this file that we want to build by default:
#
group("assembly") {
  testonly = true
  deps = [
    ":images",
    ":input_validation_checks",
    ":output_validation_checks",
  ]

  # Recovery-only configurations don't have a properly configured fuchsia.zbi,
  # as they are really building a recovery.zbi, with it's own separate
  # configuration.
  #
  # `ffx assembly` cannot build the configurations these fuchsia.zbi's have, and
  # so we clear the deps here to disable all these targets.
  if (recovery_only) {
    deps = []
  }
}

#####
#
# Labels used internally in this file
labels = {
  kernel_image_metadata = "kernel_image.gn_meta.json"
  kernel_cmdline_args_metadata = "kernel_cmdline_args.gn_meta.json"
  bootfs_entries_metadata = "bootfs_entries.gn_meta.json"
  board_config = "board_config.json"
}

#####
#
# Labels whose outputs are used by Image Assembly, and are deps in this file
dep_labels = {
  ####
  # External labels that are deps.
  #
  version_file = "//build/info:latest-commit-date"
  ota_epoch_file = "//src/sys/pkg/bin/system-updater:epoch.json"
  kernel_image = "//zircon/kernel"
  kernel_cmdline_args = "//build/input:bootfs"
  bootfs_entries = "//build/input:bootfs"

  ####
  # External labels whose outputs are compared against for validity (temporary)
  #
  system_image_pkg = "//build/images:system_image.meta"

  if (build_info_product != "bringup") {
    fuchsia_zbi = "//build/images:fuchsia"
    fuchsia_zbi_manifest = "//build/images:fuchsia_unverified"

    if (use_vbmeta) {
      fuchsia_vbmeta = "//build/images:fuchsia.vbmeta"
    }
  } else {
    fuchsia_zbi = "//build/images/bringup"
    fuchsia_zbi_manifest = fuchsia_zbi

    if (use_vbmeta) {
      fuchsia_vbmeta = "//build/images/bringup:bringup.vbmeta"
    }
  }
  fuchsia_update = "//build/images:update.meta"

  ####
  # Internal labels, in "dep" format:
  #
  kernel_image_metadata = ":" + labels.kernel_image_metadata
  kernel_cmdline_args_metadata = ":" + labels.kernel_cmdline_args_metadata
  bootfs_entries_metadata = ":" + labels.bootfs_entries_metadata
  board_config = ":" + labels.board_config
}

#####
#
# File Paths used by Image Assembly
#
#  These are all the files that are inputs, outputs, or intermediates that are
#  used by the actions in this file.
#
#  files.foo = The "inputs" or "outputs" usage of the file "foo"
#  files_args.foo = The same path, but rebased on $root_build_dir, for passing
#                   as a tool argument.
#
files = {
  assembly_config = "${target_out_dir}/config.json"
  board_config = "${target_out_dir}/board_config.json"

  # Package lists for the base package
  base_package_list = "${target_out_dir}/package_lists/base"
  cache_package_list = "${target_out_dir}/package_lists/cache"
  extra_base_package_list = "${target_out_dir}/package_lists/extra_base"
  extra_base_deps_package_list =
      "${target_out_dir}/package_lists/extra_base_deps"

  # The file that contains the version
  version_file = build_info_files.version

  # The file that contains the ota backstop
  ota_epoch_file = get_label_info(dep_labels.ota_epoch_file, "target_out_dir") +
                   "/epoch.json"

  # The results of a GN metadata walk that contains all the kernel cmd arguments
  kernel_cmdline_args_metadata =
      target_out_dir + "/" + labels.kernel_cmdline_args_metadata

  # The results of a GN metadata walk that contains the kernel's `image`
  # metadata.
  kernel_image_metadata = target_out_dir + "/" + labels.kernel_image_metadata

  # The results of a GN metadata walk that contains all of the bootfs files
  bootfs_entries_metadata =
      target_out_dir + "/" + labels.bootfs_entries_metadata

  # The working directories for the ffx assembly image task.
  gendir = "${target_out_dir}/gen"
  outdir = target_out_dir

  # Intermediates used by base package creation.
  gen_base_static_packages = "${gendir}/data/static_packages"
  gen_base_cache_packages = "${gendir}/data/cache_packages"

  # The base package output.
  base_pkg = "${target_out_dir}/base.far"

  # The ZBI output
  zbi = "${target_out_dir}/fuchsia.zbi"

  # ZBI output manifest
  zbi_manifest = "${target_out_dir}/gen/zbi.json"

  # The vbmeta (if created)
  vbmeta = "${target_out_dir}/fuchsia.vbmeta"

  #####
  # Files used for validation against external label outputs (temporary)
  #

  # The base package re-extraction (temporary) output dir
  extracted_base_pkg = "${target_out_dir}/extracted_base_pkg"

  # The extracted base package contents
  extracted_base_meta_contents = "${extracted_base_pkg}/meta/contents"
  extracted_base_meta_package = "${extracted_base_pkg}/meta/package"

  # The system-image package itself (from //build/images:system-image.meta)
  system_image_pkg =
      get_label_info(dep_labels.system_image_pkg, "target_out_dir") +
      "/system_image.meta/meta.far"

  # The system-image package re-extraction (temporary) output dir
  extracted_sysimg_pkg = "${target_out_dir}/extracted_sysimg_pkg"

  # The extracted base package contents
  extracted_sysimg_meta_contents = "${extracted_sysimg_pkg}/meta/contents"
  extracted_sysimg_meta_package = "${extracted_sysimg_pkg}/meta/package"

  if (build_info_product != "bringup") {
    # The zbi created by //build/images:fuchsia
    fuchsia_zbi = "${root_build_dir}/fuchsia.zbi"
    fuchsia_zbi_manifest = "${root_build_dir}/fuchsia_unverified.zbi.json"

    # The vbmeta for fuchsia_zbi
    fuchsia_vbmeta = "${root_build_dir}/fuchsia.vbmeta"
    fuchsia_vbmeta_salt = "${fuchsia_vbmeta}.salt"
  } else {
    # The zbi created by //build/images/bringup
    fuchsia_zbi = "${root_build_dir}/bringup.zbi"
    fuchsia_zbi_manifest = "${fuchsia_zbi}.json"

    # The vbmeta for bringup.
    fuchsia_vbmeta = "${root_build_dir}/bringup.vbmeta"
    fuchsia_vbmeta_salt = "${fuchsia_vbmeta}.salt"
  }
}

#####
# File path arguments used by Image Assembly (rebased for tool usage)
#
# This are most of the above files, rebased on root_build_dir, for use when
# calling tools with the paths.
#
# This is the hard way of doing `file_args = rebase_path(files, root_build_dir)`
file_args = {
  assembly_config = rebase_path(files.assembly_config, root_build_dir)
  board_config = rebase_path(files.board_config, root_build_dir)

  base_package_list = rebase_path(files.base_package_list, root_build_dir)
  cache_package_list = rebase_path(files.cache_package_list, root_build_dir)
  extra_base_package_list =
      rebase_path(files.extra_base_package_list, root_build_dir)
  extra_base_deps_package_list =
      rebase_path(files.extra_base_deps_package_list, root_build_dir)

  version_file = rebase_path(files.version_file, root_build_dir)
  ota_epoch_file = rebase_path(files.ota_epoch_file, root_build_dir)
  kernel_cmdline_args_metadata =
      rebase_path(files.kernel_cmdline_args_metadata, root_build_dir)
  kernel_image_metadata =
      rebase_path(files.kernel_image_metadata, root_build_dir)
  bootfs_entries_metadata =
      rebase_path(files.bootfs_entries_metadata, root_build_dir)

  gendir = rebase_path(files.gendir, root_build_dir)
  outdir = rebase_path(files.outdir, root_build_dir)

  gen_base_static_packages =
      rebase_path(files.gen_base_static_packages, root_build_dir)
  gen_base_cache_packages =
      rebase_path(files.gen_base_cache_packages, root_build_dir)

  base_pkg = rebase_path(files.base_pkg, root_build_dir)
  zbi = rebase_path(files.zbi, root_build_dir)
  zbi_manifest = rebase_path(files.zbi_manifest, root_build_dir)

  #####
  # Files used for validation against external label outputs (temporary)
  #

  #####
  # Files used for validation against external label outputs (temporary)
  #

  extracted_base_pkg = rebase_path(files.extracted_base_pkg, root_build_dir)
  extracted_base_meta_contents =
      rebase_path(files.extracted_base_meta_contents, root_build_dir)
  extracted_base_meta_package =
      rebase_path(files.extracted_base_meta_package, root_build_dir)

  system_image_pkg = rebase_path(files.system_image_pkg, root_build_dir)
  extracted_sysimg_pkg = rebase_path(files.extracted_sysimg_pkg, root_build_dir)
  extracted_sysimg_meta_contents =
      rebase_path(files.extracted_sysimg_meta_contents, root_build_dir)
  extracted_sysimg_meta_package =
      rebase_path(files.extracted_sysimg_meta_package, root_build_dir)

  fuchsia_zbi = rebase_path(files.fuchsia_zbi, root_build_dir)

  if (defined(files.fuchsia_vbmeta)) {
    fuchsia_vbmeta = rebase_path(files.fuchsia_vbmeta, root_build_dir)
    fuchsia_vbmeta_salt = rebase_path(files.fuchsia_vbmeta_salt, root_build_dir)
  }
}

#####
# These are the package groups that are used to build the system:
#
# - `base_packages` - This is the "base package set", the set of packages that
#                     are considered for all OTA updates and part of the flash
#                     files' blobfs.
#
# - `cache_packages` - These are the (usually) ephemeral packages which are pre-
#                      installed on the system, present in flash files, but not
#                      in the update package (except that they currently are
#                      placed into the `base package set` for all intents and
#                      purposes.
#
# - `universe_packages` - The "known universe of packages at build time".  These
#                         are all the packages that are defined in the build as
#                         potentially placed on the device, and therefore they
#                         need to have their shell command and config-data
#                         entries placed in base.
#
# - `meta_packages` - These are packages that exist because they are assembly-
#                     time constructed containers of data about other packages:
#                       - shell command entries
#                       - config data
#                       - indexes
#

group("base_packages") {
  testonly = base_cache_packages_testonly
  visibility = [ ":*" ]
  public_deps = [
    "//:additional_base_driver_packages",
    "//:additional_base_packages",
  ]
}

group("cache_packages") {
  testonly = base_cache_packages_testonly
  visibility = [ ":*" ]
  public_deps = [ "//:additional_cache_packages" ]
}

group("universe_packages") {
  testonly = fuchsia_zbi_testonly
  visibility = [ ":*" ]
  public_deps = [
    ":base_packages",
    ":cache_packages",
    "//:additional_universe_packages",
  ]
}

group("meta_packages") {
  visibility = [ ":*" ]
  testonly = fuchsia_zbi_testonly
  public_deps = [ "//build/images:driver-manager-base-config" ]
  public_deps += meta_package_labels
}

if (uses_base_package) {
  #####
  #  Set up the Inputs for `base` package creation.
  #
  # The above groups of packages need to be walked for metadata to find all of the
  # package manifests for the targets in those dependency trees.  These groups
  # don't include the special packages that are synthesized by the build itself
  # (those are in 'meta_packages')
  #
  # These lists of packages manifests then need to be incorporated into the JSON
  # configuration that is expected by the `ffx assembly` tooling.
  #
  #  - `base` package set
  #  - `cache` package set
  #  - `meta` package set (packages created by the build)
  #  - `extra_base_packages` package set (the packages whose contents are
  #     incorporated into the `base` package contents, not as packages that are
  #     individually listed in the base package set)
  #
  #  The following target names are used:
  #
  #  `package_manifest_list.[base, cache, extra_base]` -> These are the raw lists
  #       of package manifests that are gathered via metadata walks
  #
  #  `assembly_config_input.[base, cache, extra_base]` -> These are the converted
  #       versions of the lists into the format as expected by the tooling.
  #

  #####
  # Helper template to create the package list inputs.
  #
  template("list_package_manifests") {
    generated_file(target_name) {
      data_keys = [ "package_output_manifests" ]
      walk_keys = [ "package_barrier" ]
      outputs = [ invoker.filename ]
      output_conversion = "json"
      rebase = root_build_dir

      forward_variables_from(invoker,
                             [
                               "deps",
                               "testonly",
                             ])
    }
  }

  list_package_manifests("package_manifest_list.base") {
    testonly = true
    filename = files.base_package_list
    deps = [
      ":base_packages",
      ":meta_packages",
    ]
  }

  list_package_manifests("package_manifest_list.cache") {
    testonly = true
    filename = files.cache_package_list
    deps = [ ":cache_packages" ]
  }

  list_package_manifests("package_manifest_list.extra_base") {
    testonly = true
    filename = files.extra_base_package_list
    deps = [ "//build/input:system_image" ]
  }

  generated_file("package_manifest_list.extra_base_deps") {
    testonly = true
    data_keys = [ "system_image_extra_package_manifest" ]
    walk_keys = [ "system_image_extra_package_manifest_barrier" ]
    outputs = [ files.extra_base_deps_package_list ]
    output_conversion = "json"
    deps = [
      ":base_packages",
      ":meta_packages",
    ]
  }
}

#####
# Setup the inputs for the ZBI creation
#
#
generated_file(labels.kernel_image_metadata) {
  data_keys = [ "images" ]
  outputs = [ files.kernel_image_metadata ]
  output_conversion = "json"
  deps = [ dep_labels.kernel_image ]
}

generated_file(labels.kernel_cmdline_args_metadata) {
  testonly = true
  data_keys = [ "zbi_config_entry" ]
  walk_keys = [ "zbi_input_barrier" ]
  outputs = [ files.kernel_cmdline_args_metadata ]
  output_conversion = "json"
  deps = [ dep_labels.kernel_cmdline_args ]
}

if (bootfs_only) {
  # Bringup has an empty devmgr config.
  resource("bringup_devmgr_config.txt") {
    testonly = true
    sources = [ "bringup_devmgr_config.txt" ]
    outputs = [ "config/devmgr" ]
  }
}

generated_file(labels.bootfs_entries_metadata) {
  testonly = true
  data_keys = [ "distribution_entries" ]
  walk_keys = [ "distribution_entries_barrier" ]
  outputs = [ files.bootfs_entries_metadata ]
  output_conversion = "json"
  deps = [ dep_labels.bootfs_entries ]
  if (uses_base_package) {
    # Use the devmgr config for pkgfs
    deps += [ "//build/images:devmgr_config" ]
  } else {
    # In a bootfs-only build, all packages need to be included in bootfs, and so
    # must be added here, along with the empty devmgr config.
    deps += [
      ":bringup_devmgr_config.txt",
      ":meta_packages",
      ":universe_packages",
      "//build/input:system_image",
    ]
  }
}

#####
# Create the configuration file for the assembly tool
#
#
action("create_config") {
  testonly = true
  script = "make_assembly_config.py"
  inputs = []
  deps = []
  args = []

  # If we have a base package (the usual case), then add those inputs, deps, and
  # args to the config creation.
  if (uses_base_package) {
    inputs += [
      files.base_package_list,
      files.cache_package_list,
      files.extra_base_package_list,
      files.extra_base_deps_package_list,
      files.version_file,
      files.ota_epoch_file,
    ]

    deps += [
      ":package_manifest_list.base",
      ":package_manifest_list.cache",
      ":package_manifest_list.extra_base",
      ":package_manifest_list.extra_base_deps",
      dep_labels.version_file,
      dep_labels.ota_epoch_file,
    ]

    args += [
      "--base-packages-list",
      file_args.base_package_list,
      "--cache-packages-list",
      file_args.cache_package_list,
      "--extra-files-packages-list",
      file_args.extra_base_package_list,
      "--extra-deps-files-packages-list",
      file_args.extra_base_deps_package_list,
      "--version-file",
      file_args.version_file,
      "--epoch-file",
      file_args.ota_epoch_file,
    ]
  }

  # All builds must use a ZBI, and those relevant inputs, deps, and args:
  inputs += [
    files.kernel_image_metadata,
    files.kernel_cmdline_args_metadata,
    files.bootfs_entries_metadata,
  ]
  deps += [
    dep_labels.kernel_image_metadata,
    dep_labels.kernel_cmdline_args_metadata,
    dep_labels.bootfs_entries_metadata,
  ]

  args += [
    "--kernel-image-metadata",
    file_args.kernel_image_metadata,
    "--zbi-config-entries",
    file_args.kernel_cmdline_args_metadata,
    "--bootfs-entries",
    file_args.bootfs_entries_metadata,
    "--output",
    file_args.assembly_config,
  ]

  outputs = [ files.assembly_config ]
}

#####
# Generate the board config
#
# This is all generated from GN args.  Some values are stubs
#
generated_file(labels.board_config) {
  testonly = true
  public_deps = []
  outputs = [ files.board_config ]
  output_conversion = "json"
  contents = {
    board_name = board_name
    arch = target_cpu
    zbi = {
      partition = "zircon_a"
      name = "fuchsia"
      max_size = 0
      compression = zbi_compression
    }
    if (use_vbmeta) {
      vbmeta = {
        partition = "vbmeta_a"
        kernel_partition = "zircon"
        key = rebase_path(avb_key, root_build_dir)
        key_metadata = rebase_path(avb_atx_metadata, root_build_dir)
        additional_descriptor_files =
            rebase_path(board_extra_vbmeta_descriptors, root_build_dir)

        # TODO(aaronwood) - Remove when no longer matching against other impl.
        salt = file_args.fuchsia_vbmeta_salt
      }
    }
    fvm = {
      partition = "stub"
    }
  }

  # If vbmeta was used, and a salt file is defined, add the fuchsia vbmeta
  # label to deps.
  if (use_vbmeta) {
    public_deps += [ dep_labels.fuchsia_vbmeta ]
  }
}

#####
# Create the images themselves using `ffx assembly`
#
#
ffx_action("images") {
  # TODO(fxbug.dev/77290) - Add depfile support and then remove the following
  hermetic_deps = false

  testonly = true
  deps = [
    ":create_config",
    dep_labels.board_config,
  ]

  args = [
    "--config",
    "assembly_enabled=true",
    "assembly",
    "image",
    "--product",
    file_args.assembly_config,
    "--board",
    file_args.board_config,
    "--gendir",
    file_args.gendir,
    "--outdir",
    file_args.outdir,
  ]

  inputs = [
    files.assembly_config,
    files.board_config,
  ]

  outputs = []

  # Base package outputs, if this configuration uses them
  if (uses_base_package) {
    outputs += [
      files.base_pkg,

      # intermediate outputs that are in gendir:
      files.gen_base_static_packages,
      files.gen_base_cache_packages,
    ]
  }

  # zbi outputs
  outputs += [
    files.zbi,
    files.zbi_manifest,
  ]

  if (use_vbmeta) {
    inputs += [ files.fuchsia_vbmeta_salt ]
    outputs += [ files.vbmeta ]
  }
}

#####
#
# Validation that the inputs generated for the ffx assembly match those of the
# current implementation in //build/images/BUILD.gn
#
# NOTE: For consistency, the //build/images/BUILD.gn inputs and outputs are
#       always listed as the "first" file, and the files generated in this
#       BUILD.gn file are listed as the "second" file, to keep the diff view
#       of the files consistent (left=first=original, right=second=new)
#

group("input_validation_checks") {
  testonly = true
  deps = []
  if (uses_base_package) {
    deps += [
      ":cache_index.matches",
      ":pkgsvr_index.matches",
    ]
  }
}

if (uses_base_package) {
  verify_files_match("pkgsvr_index.matches") {
    testonly = true
    deps = [
      ":images",
      "//build/images:pkgsvr_index",
    ]

    first = "$root_build_dir/obj/build/images/pkgsvr_index"
    second = files.gen_base_static_packages

    display_text_diff_on_failure = true
  }

  verify_files_match("cache_index.matches") {
    testonly = true
    deps = [
      ":images",
      "//build/images:cache_index",
    ]

    first = "$root_build_dir/obj/build/images/cache_index"
    second = files.gen_base_cache_packages

    display_text_diff_on_failure = true
  }
}

#####
#
# Validation checks that the base package contents matches the system-image
# package contents

group("output_validation_checks") {
  testonly = true
  deps = []
  if (uses_base_package) {
    deps += [
      ":base_pkg_meta_contents_matches",
      ":base_pkg_meta_far_matches",
    ]
  }
  deps += [
    ":zbi_manifest_matches",
    ":zbi_matches",
  ]
  if (use_vbmeta) {
    deps += [ ":vbmeta_matches" ]
  }
}

if (uses_base_package) {
  # Extract system-image's package's contents for inspection
  compiled_action("extract_system_image_pkg") {
    testonly = true
    tool = "//src/sys/pkg/bin/far:bin"
    tool_output_name = "far"

    args = [
      "extract",
      "--archive=${file_args.system_image_pkg}",
      "--output=${file_args.extracted_sysimg_pkg}",
    ]

    inputs = [ files.system_image_pkg ]

    outputs = [
      files.extracted_sysimg_meta_contents,
      files.extracted_sysimg_meta_package,
    ]

    deps = [ "//build/images:system_image.meta" ]
  }

  # Extract base package's contents for inspection
  compiled_action("extract_base_pkg") {
    testonly = true
    tool = "//src/sys/pkg/bin/far:bin"
    tool_output_name = "far"

    args = [
      "extract",
      "--archive=${file_args.base_pkg}",
      "--output=${file_args.extracted_base_pkg}",
    ]

    inputs = [ files.base_pkg ]

    outputs = [
      files.extracted_base_meta_contents,
      files.extracted_base_meta_package,
    ]

    deps = [ ":images" ]
  }

  verify_files_match("base_pkg_meta_contents_matches") {
    testonly = true
    deps = [
      ":extract_base_pkg",
      ":extract_system_image_pkg",
    ]

    first = files.extracted_sysimg_meta_contents
    second = files.extracted_base_meta_contents

    display_text_diff_on_failure = true
  }

  verify_files_match("base_pkg_meta_far_matches") {
    testonly = true
    deps = [
      ":images",
      "//build/images:system_image.meta",
    ]

    first = files.system_image_pkg
    second = files.base_pkg
  }
}

verify_files_match("zbi_matches") {
  testonly = true
  deps = [
    ":images",
    dep_labels.fuchsia_zbi,
  ]

  first = files.zbi
  second = files.fuchsia_zbi
}

verify_files_match("zbi_manifest_matches") {
  testonly = true
  deps = [
    ":images",
    dep_labels.fuchsia_zbi_manifest,
  ]

  first = files.fuchsia_zbi_manifest
  second = files.zbi_manifest

  display_text_diff_on_failure = true
}

if (use_vbmeta) {
  verify_files_match("vbmeta_matches") {
    testonly = true
    deps = [
      ":images",
      dep_labels.fuchsia_vbmeta,
    ]

    first = files.fuchsia_vbmeta
    second = files.vbmeta
  }
}