build/testing/golden_test.gni

# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import("//build/testing/config.gni")
import("//build/testing/host_test.gni")
import("//build/testing/host_test_data.gni")

# Declares a golden test.
#
# This template creates a test to verify that certain generated files match
# golden copies checked into the source tree. When invoked with the --regen
# flag, the test instead overwrites the golden files with freshly generated
# copies, guaranteeing that the next test run will pass.
#
# Set the `bless_goldens` build arg to true to include `--regen` in the arguments
# to the test script.
#
# For simpler, single-file golden testing that executes during the build, see
# //build/testing/golden_file.gni.
#
# Example
# ```
# golden_test("foo_golden_tests") {
#   goldens_dir = "goldens"
#   deps = [ ":generate_json" ]
#   reformat_goldens_bin = "path/to/my/json/formatter"
#   reformat_goldens_args = [ "--pretty" ]
#   reformat_goldens_inputs = [ "path/to/my/json/formatter.config" ]
#   entries = [
#     {
#       golden = "out.json.golden"
#       generated = "$target_gen_dir/foo_out.json"
#     },
#   ]
# }
# ```
#
# Setup
# ```
# $ $EDITOR BUILD.gn  # add the example code above
# $ mkdir goldens
# $ touch goldens/goldens.txt
# $ fx test foo_golden_tests -- --regen
# Regenerating goldens in /path/to/your/goldens
# ...
# $ fx test foo_golden_tests
# ...
# PASS
# ```
#
# Parameters
#
#   goldens_dir (required)
#     Directory of golden files, relative to the invoking BUILD.gn file. Must
#     contain a goldens.txt file. Each file path listed in goldens.txt must
#     exist, otherwise a gen-time error is produced.
#     Type: path
#
#   entries (required)
#     List of scopes, each with `golden` and `generated` fields. The `golden`
#     field is a file path relative to `goldens_dir`. The `generated` field is
#     an absolute path to the corresponding generated file, which must be
#     produced by one of the targets in `deps`.
#     Type: list(scope)
#
#   reformat_goldens_bin (optional)
#     A command that will reformat from stdin to stdout.
#     The command will be invoked for each golden file checked into the tree
#     before comparing it to the newly generated version. This allows changes
#     to source formatting tools to not cause goldens tests to fail.
#     Type: string
#
#   reformat_goldens_args (optional)
#     A list of args for invoking `reformat_goldens_bin`.
#     Type: list(string)
#     Default: empty
#
#   reformat_goldens_inputs (optional)
#     Inputs the reformat_goldens_command uses, this should typically include
#     a config file the formatter uses, if any. This makes sure when the config
#     changes, this template can correctly rerun the reformat action.
#     Type: list(string)
#     Default: empty
#
#   deps (required)
#   visibility
#
template("golden_test") {
  assert(is_host, "The golden_test template can only be used on host")

  assert(defined(invoker.goldens_dir))
  assert(defined(invoker.entries))
  assert(defined(invoker.deps))

  copied_goldens_dir = "${target_gen_dir}/${target_name}"
  manifest_file = "${target_gen_dir}/${target_name}_manifest.json"

  # Read the checked-in list of golden files.
  goldens_list = read_file("${invoker.goldens_dir}/goldens.txt", "list lines")
  assert(goldens_list ==
             process_file_template(goldens_list, "{{source_file_part}}"),
         "Subdirectories are not allowed in goldens.txt")

  # Copy the goldens.txt list and make it available to the test at runtime.
  target_test_data_goldens_list = "${target_name}_test_data_goldens_list"
  host_test_data(target_test_data_goldens_list) {
    visibility = [ ":*" ]
    sources = [ "${invoker.goldens_dir}/goldens.txt" ]
    outputs = [ "${copied_goldens_dir}/goldens.txt" ]
  }

  # Reformat or copy the goldens from the source directory by piping each
  # golden file through `reformat_goldens`.
  target_reformat_goldens = "${target_name}_reformat_goldens"
  if (goldens_list == []) {
    # No-op target to avoid breaking `gn gen` when goldens.txt is empty
    # (action_foreach is not allowed to have zero source files).
    group(target_reformat_goldens) {
    }
    not_needed(invoker,
               [
                 "reformat_goldens_bin",
                 "reformat_goldens_args",
                 "reformat_goldens_inputs",
               ])
  } else {
    action_foreach(target_reformat_goldens) {
      testonly = true
      visibility = [ ":*" ]

      # Always copy & format the goldens when the generated files change to avoid
      # comparing stale goldens.
      deps = invoker.deps

      script = "//build/testing/reformat_goldens.py"
      sources = rebase_path(goldens_list, ".", invoker.goldens_dir)
      inputs = []
      if (defined(invoker.reformat_goldens_inputs)) {
        inputs += invoker.reformat_goldens_inputs
      }

      outputs = [ "${copied_goldens_dir}/{{source_file_part}}" ]
      output_dir = rebase_path(copied_goldens_dir, root_build_dir)

      args = [
        "--source",
        "{{source}}",
        "--output",
        "${output_dir}/{{source_file_part}}",
      ]
      if (defined(invoker.reformat_goldens_bin)) {
        inputs += [ invoker.reformat_goldens_bin ]
        args += [
          "--formatter",
          rebase_path(invoker.reformat_goldens_bin, root_build_dir),
        ]
        if (defined(invoker.reformat_goldens_args)) {
          args += [ "--" ] + invoker.reformat_goldens_args
        }
      } else {
        assert(
            !defined(invoker.reformat_goldens_args),
            "reformat_golden_args is only valid if a reformat_goldens_bin is specified")
      }
    }
  }

  # Make the copy of the goldens available to the test at runtime.
  target_test_data_goldens = "${target_name}_test_data_goldens"
  host_test_data(target_test_data_goldens) {
    visibility = [ ":*" ]
    deps = [ ":${target_reformat_goldens}" ]
    sources = []
    foreach(golden, goldens_list) {
      sources += [ "${copied_goldens_dir}/${golden}" ]
    }
  }

  # Generate the manifest file.
  target_generate_manifest = "${target_name}_generate_manifest"
  generated_file(target_generate_manifest) {
    visibility = [ ":*" ]
    testonly = true
    contents = {
      test_goldens_dir = rebase_path(copied_goldens_dir, root_build_dir)
      regen_goldens_dir = rebase_path(invoker.goldens_dir, root_build_dir)
      entries = []
    }
    foreach(entry, invoker.entries) {
      assert(get_path_info(entry.golden, "extension") == "golden",
             "Golden filename '${entry.golden}' does not end in '.golden'")
      contents.entries += [
        {
          golden = entry.golden
          generated = rebase_path(entry.generated, root_build_dir)
        },
      ]
    }
    outputs = [ manifest_file ]
    output_conversion = "json"
  }

  # Allow the test to access the manifest file and generated files.
  target_generated_files = "${target_name}_generated_files"
  host_test_data(target_generated_files) {
    visibility = [ ":*" ]
    sources = [ manifest_file ]
    foreach(entry, invoker.entries) {
      sources += [ entry.generated ]
    }
    deps = [ ":${target_generate_manifest}" ] + invoker.deps
  }

  # Generate a test target, so that this test can be executed with `fx test`.
  host_test(target_name) {
    forward_variables_from(invoker, [ "visibility" ])
    binary_path = "${root_out_dir}/golden-util"
    args = [
      "--manifest",
      rebase_path(manifest_file, root_build_dir),
    ]
    if (bless_goldens) {
      args += [ "--regen" ]
    }
    deps = [
      ":${target_generated_files}",
      ":${target_test_data_goldens_list}",
      ":${target_test_data_goldens}",
      "//tools/golden-util",
    ]
  }
}