src/storage/fshost/integration/BUILD.gn

# Copyright 2021 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import("//build/components.gni")
import("//build/rust/rustc_library.gni")
import("//build/rust/rustc_test.gni")
import("//src/storage/fshost/fshost.gni")
import("//tools/cmc/build/cmc.gni")

rustc_library("fshost_test_fixture") {
  edition = "2021"
  testonly = true
  sources = [
    "src/disk_builder.rs",
    "src/fshost_builder.rs",
    "src/lib.rs",
    "src/mocks.rs",
  ]
  deps = [
    "//sdk/fidl/fuchsia.boot:fuchsia.boot_rust",
    "//sdk/fidl/fuchsia.component.test:fuchsia.component.test_rust",
    "//sdk/fidl/fuchsia.driver.test:fuchsia.driver.test_rust",
    "//sdk/fidl/fuchsia.feedback:fuchsia.feedback_rust",
    "//sdk/fidl/fuchsia.fshost:fuchsia.fshost_rust",
    "//sdk/fidl/fuchsia.hardware.block.volume:fuchsia.hardware.block.volume_rust",
    "//sdk/fidl/fuchsia.hardware.ramdisk:fuchsia.hardware.ramdisk_rust",
    "//sdk/fidl/fuchsia.io:fuchsia.io_rust",
    "//sdk/fidl/fuchsia.logger:fuchsia.logger_rust",
    "//sdk/fidl/fuchsia.process:fuchsia.process_rust",
    "//sdk/fidl/fuchsia.security.keymint:fuchsia.security.keymint_rust",
    "//sdk/fidl/fuchsia.storage.partitions:fuchsia.storage.partitions_rust",
    "//sdk/lib/driver_test_realm/realm_builder/rust",
    "//sdk/rust/zx",
    "//src/devices/bind/fuchsia.platform:fuchsia.platform_rust",
    "//src/devices/bus/drivers/platform",
    "//src/lib/diagnostics/reader/rust",
    "//src/lib/diagnostics/testing/diagnostics-assertions/rust:diagnostics-assertions",
    "//src/lib/fidl/rust/fidl",
    "//src/lib/fuchsia-async",
    "//src/lib/fuchsia-component",
    "//src/lib/fuchsia-component-test",
    "//src/lib/fuchsia-fs",
    "//src/security/lib/fake-keymint",
    "//src/storage/blobfs/bin:blobfs-component",
    "//src/storage/crypt/policy",
    "//src/storage/crypt/zxcrypt",
    "//src/storage/fidl/fuchsia.fs.startup:fuchsia.fs.startup_rust",
    "//src/storage/fshost:fuchsia.fshost.fxfsprovisioner_rust",
    "//src/storage/fshost/config",
    "//src/storage/fxfs/fidl/fuchsia.fxfs:fuchsia.fxfs_rust",
    "//src/storage/gpt/component:lib",
    "//src/storage/gpt/rust",
    "//src/storage/lib/blob_writer:blob-writer",
    "//src/storage/lib/block_client/rust",
    "//src/storage/lib/block_server:vmo_backed_block_server",
    "//src/storage/lib/delivery_blob",
    "//src/storage/lib/fs_management/rust:fs_management",
    "//src/storage/lib/key-bag",
    "//src/storage/lib/ramdevice_client/rust",
    "//src/storage/lib/vfs/rust:vfs",
    "//src/storage/minfs/bin:minfs-component",
    "//src/storage/testing:storage_drivers",
    "//src/storage/testing/rust:storage-isolated-driver-manager",
    "//src/sys/lib/cm_rust",
    "//src/sys/pkg/fidl/fuchsia.update.verify:fuchsia.update.verify_rust",
    "//src/sys/pkg/lib/fuchsia-hash",
    "//src/sys/pkg/lib/fuchsia-merkle",
    "//third_party/rust_crates:anyhow",
    "//third_party/rust_crates:assert_matches",
    "//third_party/rust_crates:futures",
    "//third_party/rust_crates:log",
    "//third_party/rust_crates:serde_json",
    "//third_party/rust_crates:uuid",
    "//third_party/rust_crates:zerocopy",
    "//third_party/rust_crates:zstd",
    "//third_party/zstd",
  ]

  data_deps = [ "//src/storage/crypt/fxfs" ]
}

# Generates an fshost integration test component based on one of the integration test files. The
# binary also has config.rs included, and a couple of default dependencies, but the rest should be
# specified as needed for the tests in the file.
#
#   data_filesystem_variant (required)
#     [string] A variant string passed to the integration test harness.
#
#   data_filesystem_format (required)
#     [string] The format of the data partition fshost exports. One of "fxfs", "minfs", or "f2fs".
#
#   fshost_target (required)
#     [string] The target for the fshost under test.
#
#   source_root (required)
#     [string] The main file for the integration test. Likely one of the files under "tests".
#
#   deps (optional)
#     [array] Dependencies for the rust test binary.
#
#   extra_features (optional)
#     [array] Extra rust features to define.
#
template("fshost_integration_test_component") {
  _fshost_component_name = get_label_info(invoker.fshost_target, "name")
  _test_bin = string_replace(target_name, "-", "_")
  _test_bin = "${_test_bin}_bin"
  rustc_test(_test_bin) {
    edition = "2021"
    source_root = invoker.source_root
    sources = [
      "tests/config.rs",
      invoker.source_root,
    ]
    rustenv = [
      "FSHOST_COMPONENT_NAME=${_fshost_component_name}",
      "DATA_FILESYSTEM_FORMAT=${invoker.data_filesystem_format}",
      "DATA_FILESYSTEM_VARIANT=${invoker.data_filesystem_variant}",
    ]
    features = [
      invoker.data_filesystem_format,
      invoker.data_filesystem_variant,
    ]
    if (defined(invoker.extra_features)) {
      features += invoker.extra_features
    }
    deps = [
      ":fshost_test_fixture",
      "//src/lib/fuchsia",
    ]
    if (defined(invoker.deps)) {
      deps += invoker.deps
    }
  }

  _shard_target = "${target_name}-shard"
  generated_file(_shard_target) {
    testonly = true
    contents = {
      program = {
        binary = "bin/${_test_bin}"
      }
    }
    outputs = [ "${target_out_dir}/${_shard_target}.cml" ]
    output_conversion = "json"
    visibility = [ ":*" ]
  }
  _shard = get_target_outputs(":$_shard_target")
  _shard = _shard[0]

  _manifest_target = "${target_name}-manifest.cml"
  cmc_merge("$_manifest_target") {
    testonly = true
    sources = [
      "meta/fshost_integration_test.cml",
      _shard,
    ]
    deps = [ ":$_shard_target" ]
  }
  _manifest = get_target_outputs(":$_manifest_target")
  _manifest = _manifest[0]

  fuchsia_test_component(target_name) {
    manifest = _manifest
    deps = [
      ":$_manifest_target",
      ":$_test_bin",
    ]
  }
}

# Generates an fshost integration test package. Includes the fshost component on which the test is
# executed.
#
#   enable_migration_tests (default: true)
#     [bool] Include migration tests in the integration test package.
#
#   enable_corruption_tests (default: true)
#     [bool] Include corruption tests in the integration test package.
#
#   enable_system_volume_manipulation_tests (default: true)
#     [bool] Include tests for fuchsia.fshost/Recovery methods that manipulate volumes in the
#            system container.
#
#   enable_provision_fxfs_tests (default: false)
#     [bool] Include provision fxfs tests in the integration test package.
#
#   data_filesystem_variant (optional)
#     [string] A variant string passed to the integration test harness. Defaults to the
#              data_filesystem_format.
#
#   options.data_filesystem_format (required)
#     [string] The format of the data partition fshost exports. One of "fxfs", "minfs", or "f2fs".
#              The options scope can also contain additional fshost options if required.
#
#   options.storage_host (optional)
#     [bool] Whether storage-host should be used.
#
#   extra_deps (optional)
#     [array] A list of extra deps for the general integration test component. These deps are not
#             added to any of the other components right now. NB: if other test components need
#             extra deps then come up with a different solution here.
#
template("fshost_integration_test_package") {
  assert(defined(invoker.options), "fshost options not specified")
  fshost_options = invoker.options
  if (!defined(invoker.data_filesystem_variant)) {
    invoker.data_filesystem_variant = fshost_options.data_filesystem_format
  }
  if (!defined(invoker.enable_migration_tests)) {
    invoker.enable_migration_tests = true
  }
  if (!defined(invoker.enable_corruption_tests)) {
    invoker.enable_corruption_tests = true
  }
  if (!defined(invoker.enable_system_volume_manipulation_tests)) {
    invoker.enable_system_volume_manipulation_tests = false
  }
  if (!defined(fshost_options.data_filesystem_format)) {
    # data_filesystem_format is implied from fxblob.
    assert(!(defined(fshost_options.fxfs_blob) && !fshost_options.fxfs_blob))
    fshost_options.data_filesystem_format = "fxfs"
  }

  _fshost_target = "test-fshost-${target_name}"
  fshost_component_and_config(_fshost_target) {
    testonly = true
    options = invoker.options
    if (!defined(options.use_disk_migration)) {
      options.use_disk_migration = true
    }
  }

  _extra_features = []
  if (defined(fshost_options.storage_host) && fshost_options.storage_host) {
    _extra_features = [ "storage-host" ]
  }

  _test_packages = []

  #
  # General Integration Tests
  #
  _integration_component_target = "${target_name}_general"
  _test_components = [ ":$_integration_component_target" ]
  fshost_integration_test_component(_integration_component_target) {
    fshost_target = _fshost_target
    data_filesystem_format = fshost_options.data_filesystem_format
    data_filesystem_variant = invoker.data_filesystem_variant
    extra_features = _extra_features
    source_root = "tests/fshost_integration_test.rs"
    deps = [
      "//sdk/fidl/fuchsia.fshost:fuchsia.fshost_rust",
      "//sdk/fidl/fuchsia.hardware.block.volume:fuchsia.hardware.block.volume_rust",
      "//sdk/fidl/fuchsia.io:fuchsia.io_rust",
      "//sdk/lib/device-watcher/rust",
      "//sdk/rust/zx",
      "//src/lib/fidl/rust/fidl",
      "//src/lib/fuchsia-async",
      "//src/lib/fuchsia-component",
      "//src/lib/fuchsia-fs",
      "//src/storage/fidl/fuchsia.fs.startup:fuchsia.fs.startup_rust",
      "//src/storage/lib/delivery_blob",
      "//src/storage/lib/fs_management/rust:fs_management",
      "//src/sys/pkg/fidl/fuchsia.update.verify:fuchsia.update.verify_rust",
      "//src/sys/pkg/lib/fuchsia-merkle",
      "//third_party/rust_crates:assert_matches",
      "//third_party/rust_crates:futures",
      "//third_party/rust_crates:log",
      "//third_party/rust_crates:regex",
    ]
    if (defined(invoker.extra_deps)) {
      deps += invoker.extra_deps
    }
    if (data_filesystem_variant == "fxblob") {
      deps += [
        "//sdk/fidl/fuchsia.fs:fuchsia.fs_rust",
        "//src/storage/crypt/policy",
        "//src/storage/fxfs/fidl/fuchsia.fxfs:fuchsia.fxfs_rust",
        "//src/storage/lib/blob_writer:blob-writer",
      ]
    }
    if (defined(fshost_options.storage_host) && fshost_options.storage_host) {
      deps += [
        "//sdk/fidl/fuchsia.hardware.block.partition:fuchsia.hardware.block.partition_rust",
        "//sdk/fidl/fuchsia.storage.partitions:fuchsia.storage.partitions_rust",
      ]
    }
  }

  #
  # Migration Tests
  #
  if (invoker.enable_migration_tests) {
    _migration_component_target = "${target_name}_migration"
    _test_components += [ ":$_migration_component_target" ]
    fshost_integration_test_component(_migration_component_target) {
      fshost_target = _fshost_target
      data_filesystem_format = fshost_options.data_filesystem_format
      data_filesystem_variant = invoker.data_filesystem_variant
      extra_features = _extra_features
      source_root = "tests/migration.rs"
    }
  }

  #
  # System Volume Manipulation Tests
  #
  if (invoker.enable_system_volume_manipulation_tests) {
    assert(invoker.data_filesystem_variant == "fxblob")
    assert(defined(fshost_options.storage_host) && fshost_options.storage_host)
    _system_volume_manipulation_component_target =
        "${target_name}_system_volume_manipulation"
    fshost_integration_test_component(
        _system_volume_manipulation_component_target) {
      fshost_target = _fshost_target
      data_filesystem_format = fshost_options.data_filesystem_format
      data_filesystem_variant = invoker.data_filesystem_variant
      extra_features = _extra_features
      source_root = "tests/system_volume_manipulation.rs"
      deps = [
        "//sdk/fidl/fuchsia.fshost:fuchsia.fshost_rust",
        "//sdk/fidl/fuchsia.hardware.block:fuchsia.hardware.block_rust",
        "//sdk/fidl/fuchsia.io:fuchsia.io_rust",
        "//sdk/fidl/fuchsia.storage.partitions:fuchsia.storage.partitions_rust",
        "//sdk/lib/device-watcher/rust",
        "//sdk/rust/zx",
        "//src/lib/fidl/rust/fidl",
        "//src/lib/fuchsia-async",
        "//src/lib/fuchsia-component",
        "//src/lib/fuchsia-fs",
        "//src/storage/fxfs/fidl/fuchsia.fxfs:fuchsia.fxfs_rust",
        "//src/storage/fxfs/make-blob-image",
        "//src/storage/lib/block_client/rust",
        "//src/storage/lib/block_server:vmo_backed_block_server",
        "//src/storage/lib/sparse/rust:lib",
        "//src/storage/lib/storage_device",
        "//src/sys/pkg/lib/fuchsia-hash",
        "//src/sys/pkg/lib/fuchsia-merkle",
        "//third_party/rust_crates:futures",
      ]
    }

    # These tests verify mounting a corrupt or unformatted fxfs partition, which results in some
    # error logs. This requires we use a separate test package so we can control the test specs.
    _system_volume_manipulation_test_package =
        "${target_name}_system_volume_manipulation_package"
    _test_packages += [ ":$_system_volume_manipulation_test_package" ]
    fuchsia_test_package(_system_volume_manipulation_test_package) {
      package_name = "$_system_volume_manipulation_component_target"
      test_specs = {
        log_settings = {
          max_severity = "ERROR"
        }
        parallel = 1
      }
      test_components = [ ":$_system_volume_manipulation_component_target" ]
      deps = [
        ":$_fshost_target",
        "//src/storage/fxfs:fxfs_component",
      ]
    }
  }

  #
  # Write Data File Tests
  #
  _write_data_file_component_target = "${target_name}_write_data_file"
  _test_components += [ ":$_write_data_file_component_target" ]
  fshost_integration_test_component(_write_data_file_component_target) {
    fshost_target = _fshost_target
    data_filesystem_format = fshost_options.data_filesystem_format
    data_filesystem_variant = invoker.data_filesystem_variant
    extra_features = _extra_features
    source_root = "tests/write_data_file.rs"
    deps = [
      "//sdk/fidl/fuchsia.fshost:fuchsia.fshost_rust",
      "//sdk/fidl/fuchsia.io:fuchsia.io_rust",
      "//sdk/rust/zx",
      "//src/lib/fuchsia-fs",
    ]
  }

  #
  # Corruption Tests
  #
  if (invoker.enable_corruption_tests) {
    _corrupt_component_target = "${target_name}_corrupt"
    fshost_integration_test_component(_corrupt_component_target) {
      fshost_target = _fshost_target
      data_filesystem_format = fshost_options.data_filesystem_format
      data_filesystem_variant = invoker.data_filesystem_variant
      extra_features = _extra_features
      source_root = "tests/corrupt.rs"
    }

    # Corruption tests get their own package so we can give them separate test specs
    _corrupt_test_package = "${target_name}_corrupt_package"
    _test_packages += [ ":$_corrupt_test_package" ]
    fuchsia_test_package(_corrupt_test_package) {
      package_name = "$_corrupt_component_target"
      test_specs = {
        log_settings = {
          max_severity = "ERROR"
        }
        parallel = 1
      }
      test_components = [ ":$_corrupt_component_target" ]

      deps = [
        ":$_fshost_target",

        # We need to make sure we have all the filesystems available in the package for various
        # tests of the migration paths.
        "//src/storage/blobfs/bin:component",
        "//src/storage/crypt/fxfs",
        "//src/storage/f2fs/bin:f2fs-component",
        "//src/storage/fxfs:fxfs_component",
        "//src/storage/minfs/bin:minfs-component",
      ]
    }
  }

  #
  # Provision Fxfs Tests
  #
  if (defined(invoker.enable_provision_fxfs_tests) &&
      invoker.enable_provision_fxfs_tests) {
    _provision_fxfs_component_target = "${target_name}_provision_fxfs"
    fshost_integration_test_component(_provision_fxfs_component_target) {
      fshost_target = _fshost_target
      data_filesystem_format = fshost_options.data_filesystem_format
      data_filesystem_variant = invoker.data_filesystem_variant
      extra_features = _extra_features
      source_root = "tests/provision_fxfs.rs"
      deps = []
    }

    # Provision Fxfs tests get their own package so we can give them separate test specs (
    # provisioning Fxfs produces messages of ERROR severity when attempting to mount the filesystem
    # before Fxfs has been provisioned).
    #
    # TODO(https://fxbug.dev/439942311): Don't emit error messages for non-errors.
    _provision_fxfs_test_package = "${target_name}_provision_fxfs_package"
    _test_packages += [ ":$_provision_fxfs_test_package" ]
    fuchsia_test_package(_provision_fxfs_test_package) {
      package_name = "$_provision_fxfs_component_target"
      test_specs = {
        log_settings = {
          # Allow logs of up to ERROR severity.
          max_severity = "ERROR"
        }
        parallel = 1
      }
      test_components = [ ":$_provision_fxfs_component_target" ]

      deps = [ ":$_fshost_target" ]
    }
  }

  _main_test_package = "${target_name}_test_package"
  _main_test_package_name = "$target_name"
  _test_packages += [ ":$_main_test_package" ]
  fuchsia_test_package(_main_test_package) {
    package_name = _main_test_package_name
    test_specs = {
      parallel = 1
    }
    test_components = _test_components

    deps = [
      ":$_fshost_target",

      # We need to make sure we have all the filesystems available in the package for various tests
      # of the migration paths.
      "//src/storage/blobfs/bin:component",
      "//src/storage/f2fs/bin:f2fs-component",
      "//src/storage/fxfs:fxfs_component",
      "//src/storage/minfs/bin:minfs-component",
    ]
  }

  group(target_name) {
    testonly = true
    deps = _test_packages
  }
}

fshost_integration_test_package("fshost_integration_tests_fxfs") {
  # TODO(https://fxbug.dev/394970436): The corruption tests are flaky on devfs. Once we are fully on
  # storage host, we can remove this todo and the storage-host requirement, because the flaky
  # configuration will no longer exist.
  enable_corruption_tests = false
  enable_migration_tests = false
  data_filesystem_variant = "fxblob"
  options = {
  }
  extra_deps =
      [ "//src/sys/pkg/fidl/fuchsia.update.verify:fuchsia.update.verify_rust" ]
}

fshost_integration_test_package("fshost_integration_tests_storage_host_fxfs") {
  enable_migration_tests = false
  data_filesystem_variant = "fxblob"
  enable_provision_fxfs_tests = true
  enable_system_volume_manipulation_tests = true
  options = {
    use_disk_migration = false
    storage_host = true
  }
  extra_deps = [
    "//src/storage/gpt/component",
    "//src/sys/pkg/fidl/fuchsia.update.verify:fuchsia.update.verify_rust",
  ]
}

fshost_integration_test_package(
    "fshost_integration_tests_storage_host_fvm_minfs") {
  # TODO(https://fxbug.dev/339491886): Support these components in storage-host
  enable_migration_tests = false

  options = {
    data_filesystem_format = "minfs"
    use_disk_migration = false
    storage_host = true
    fxfs_blob = false
  }
  extra_deps = [
    "//src/storage/fvm:fvm-component",
    "//src/storage/gpt/component",
  ]
}

fshost_integration_test_package(
    "fshost_integration_tests_storage_host_fvm_minfs_no_zxcrypt") {
  # TODO(https://fxbug.dev/339491886): Support these components in storage-host
  enable_migration_tests = false

  data_filesystem_variant = "minfs-no-zxcrypt"
  options = {
    data_filesystem_format = "minfs"
    use_disk_migration = false
    storage_host = true
    fxfs_blob = false
    no_zxcrypt = true
  }
  extra_deps = [
    "//src/storage/fvm:fvm-component",
    "//src/storage/gpt/component",
  ]
}

fshost_integration_test_package("fshost_integration_tests_fxfs_no_fxblob") {
  # TODO(https://fxbug.dev/394970436): The corruption tests are flaky on devfs. Once we are fully on
  # storage host, we can remove this todo and the storage-host requirement, because the flaky
  # configuration will no longer exist.
  enable_corruption_tests = false
  options = {
    data_filesystem_format = "fxfs"
    fxfs_blob = false
  }
}

fshost_integration_test_package("fshost_integration_tests_f2fs") {
  # TODO(https://fxbug.dev/394970436): The corruption tests are flaky on devfs. Once we are fully on
  # storage host, we can remove this todo and the storage-host requirement, because the flaky
  # configuration will no longer exist.
  enable_corruption_tests = false
  options = {
    data_filesystem_format = "f2fs"
    fxfs_blob = false
  }
}

fshost_integration_test_package("fshost_integration_tests_minfs") {
  # TODO(https://fxbug.dev/394970436): The corruption tests are flaky on devfs. Once we are fully on
  # storage host, we can remove this todo and the storage-host requirement, because the flaky
  # configuration will no longer exist.
  enable_corruption_tests = false
  options = {
    data_filesystem_format = "minfs"
    fxfs_blob = false
  }
}

fshost_integration_test_package("fshost_integration_tests_f2fs_no_zxcrypt") {
  # TODO(https://fxbug.dev/394970436): The corruption tests are flaky on devfs. Once we are fully on
  # storage host, we can remove this todo and the storage-host requirement, because the flaky
  # configuration will no longer exist.
  enable_corruption_tests = false
  data_filesystem_variant = "f2fs-no-zxcrypt"
  options = {
    no_zxcrypt = true
    data_filesystem_format = "f2fs"
    fxfs_blob = false
  }
}

fshost_integration_test_package("fshost_integration_tests_minfs_no_zxcrypt") {
  # TODO(https://fxbug.dev/394970436): The corruption tests are flaky on devfs. Once we are fully on
  # storage host, we can remove this todo and the storage-host requirement, because the flaky
  # configuration will no longer exist.
  enable_corruption_tests = false
  data_filesystem_variant = "minfs-no-zxcrypt"
  options = {
    no_zxcrypt = true
    data_filesystem_format = "minfs"
    fxfs_blob = false
  }
}

group("tests") {
  testonly = true
  deps = [
    ":fshost_integration_tests_f2fs",
    ":fshost_integration_tests_f2fs_no_zxcrypt",
    ":fshost_integration_tests_fxfs",
    ":fshost_integration_tests_fxfs_no_fxblob",
    ":fshost_integration_tests_minfs",
    ":fshost_integration_tests_minfs_no_zxcrypt",
    ":fshost_integration_tests_storage_host_fvm_minfs",
    ":fshost_integration_tests_storage_host_fvm_minfs_no_zxcrypt",
    ":fshost_integration_tests_storage_host_fxfs",
  ]
}