tools/devshell/tests/subcommands/fx_test_test

#!/bin/bash
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

### Test expected behavior of fx test

BT_LINKED_DEPS=(
  "third_party"
  "prebuilt/third_party/dart"
  "scripts/fxtest"
  "scripts/fxutils"
)

BT_FILE_DEPS=(
  "scripts/fx"
  "tools/devshell/tests/subcommands/data/fx_test_test/tests_hashfile"
  "tools/devshell/tests/subcommands/data/fx_test_test/tests_multiple_in_package.json"
  "tools/devshell/tests/subcommands/data/fx_test_test/tests_package_server_integration.json"
  "tools/devshell/lib/fx-cmd-locator.sh"
  "tools/devshell/lib/fx-optional-features.sh"
  "tools/devshell/lib/vars.sh"
  "tools/devshell/lib/platform.sh"
  "tools/devshell/test"
)

BT_MOCKED_TOOLS=(
  "tools/devshell/build"
  "tools/devshell/is-package-server-running"
  "tools/devshell/update-if-in-base"
  "tools/devshell/shell"
  "tools/devshell/symbolize"
  "tools/devshell/lib/metrics_custom_report.sh"
)

declare fx DATA_DIR

BT_SET_UP() {
  source "${BT_TEMP_DIR}/tools/devshell/tests/lib/fuchsia-mock.sh"
  fx="$(btf::setup_fx)"
  btf::make_installed_hosttools_mock "ffx" > /dev/null
  btf::make_installed_hosttools_mock "symbolizer" > /dev/null
  btf::make_installed_hosttools_mock "symbol-index" > /dev/null
  DATA_DIR="${BT_TEMP_DIR}/tools/devshell/tests/subcommands/data/fx_test_test"
}

# Test that the "fx test --info" outputs in the format expected by other
# commands, eg `fx run-test`
TEST_fxtest_info() {
  cp "${DATA_DIR}/tests_multiple_in_package.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  BT_EXPECT ${fx} test --info --exact > "${out}"
  BT_EXPECT_EQ "$(sed -n 's/^package_url: \(.*\)/\1/p' "${out}" | wc -l)" 7
}

# Test that `fx test` calls `fx update-if-in-base` and `fx is-package-server-running` properly
TEST_fxtest_package_server_integration() {
  cp "${DATA_DIR}/tests_package_server_integration.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname="overflow_fuzzer_test"
  BT_EXPECT ${fx} test --no-use-package-hash ${testname} > ${out}
  # ensure that is-package-server-running was called
  BT_ASSERT_FILE_EXISTS "${BT_TEMP_DIR}/tools/devshell/is-package-server-running.mock_state"
  # ensure that update-if-in-base was called with the proper testname
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/update-if-in-base" "${testname}"
}

# Ensure that `fx build` is called by default
TEST_fxtest_build() {
  cp "${DATA_DIR}/tests_package_server_integration.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname="overflow_fuzzer_test"
  BT_EXPECT ${fx} test --no-use-package-hash ${testname} > ${out}
  # ensure that fx build was called
  # TODO: once fx test calls fx build with a specific target, check it here as well
  BT_ASSERT_FILE_EXISTS "${BT_TEMP_DIR}/tools/devshell/build.mock_state"
}

# Ensure that `fx test` exits with a non-zero status if `fx build` fails.
TEST_fxtest_build_failure_exits_with_nonzero_status() {
  cp "${DATA_DIR}/tests_package_server_integration.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname="overflow_fuzzer_test"
  echo 1 > "${BT_TEMP_DIR}/tools/devshell/build.mock_status"
  BT_EXPECT_FAIL ${fx} test --no-use-package-hash ${testname} > ${out}
  # ensure that fx build was called
  # TODO: once fx test calls fx build with a specific target, check it here as well
  BT_ASSERT_FILE_EXISTS "${BT_TEMP_DIR}/tools/devshell/build.mock_state"
}

# Ensure that `fx build` is not called when "--no-build" option is given
TEST_fxtest_nobuild() {
  cp "${DATA_DIR}/tests_package_server_integration.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname="overflow_fuzzer_test"
  BT_EXPECT ${fx} test --no-use-package-hash --no-build ${testname} > ${out}
  # ensure that fx build was called
  # TODO: once fx test calls fx build with a specific target, check it here as well
  BT_ASSERT_FILE_DOES_NOT_EXIST "${BT_TEMP_DIR}/tools/devshell/build.mock_state"
}

# Ensure that `fx test` exits with a non-zero status if
# `fx # is-package-server-running` fails.
TEST_fxtest_fails_with_no_package_server() {
  cp "${DATA_DIR}/tests_package_server_integration.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname1="overflow_fuzzer_test"
  echo 1 > "${BT_TEMP_DIR}/tools/devshell/is-package-server-running.mock_status"
  BT_EXPECT_FAIL ${fx} test --no-use-package-hash --no-build ${testname} > ${out}
  # ensure that fx is-package-server-running was called
  BT_ASSERT_FILE_EXISTS "${BT_TEMP_DIR}/tools/devshell/is-package-server-running.mock_state"
}

# Test that "fx test" runs a component test pinning it to the hash (merkleroot) of
# the component package, so that the user has confidence that if a test runs, it is
# running the exact same version that has been built
TEST_fxtest_hashpinnning() {
  cp -R "${DATA_DIR}/tests_hashfile/out" "${BT_TEMP_DIR}"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname1="overflow_fuzzer_test"
  local hash1="913cdd63ab4aa794694448450505efaa2a8fe27fb33888e5156da9db60ac0a29"
  local testname2="hello_world_cpp_unittests"
  local hash2="7a604498e05fa012391b6b51da9cc74ff6a6a9d25b1376de98125c194232bfa1"

  # expect that "fx shell run-test-component URL-WITH-HASH" was executed
  BT_EXPECT ${fx} test --no-build ${testname1} >> "${out}"
  local packageUrl1="fuchsia-pkg://fuchsia.com/example-fuzzers?hash=${hash1}#meta/overflow_fuzzer_test.cmx"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/shell" "run-test-component" "${packageUrl1}"

  # expect that "fx shell run-test-component URL-WITH-HASH" was executed
  BT_EXPECT ${fx} test --no-build ${testname2} >> "${out}"
  local packageUrl2="fuchsia-pkg://fuchsia.com/hello_world_cpp_tests?hash=${hash2}#meta/hello_world_cpp_unittests.cmx"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/shell.mock_state.2" "run-test-component" "${packageUrl2}"
}

# Test that when the merkle root of a package is changed as a result of the
# 'fx build' executed by fx test, the new hash is used when pinning the
# package in fx shell run-test-component
TEST_fxtest_updated_hash() {
  cp -R "${DATA_DIR}/tests_hashfile/out" "${BT_TEMP_DIR}"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname1="overflow_fuzzer_test"
  local hash_old="913cdd63ab4aa794694448450505efaa2a8fe27fb33888e5156da9db60ac0a29"
  local hash_new="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"

  # create a build script that, when called, changes hash_old to hash_new in the
  # package repository
  cat > "${BT_TEMP_DIR}/tools/devshell/build.mock_side_effects" <<EOF
  sed -i 's/${hash_old}/${hash_new}/g' ${BT_TEMP_DIR}/out/default/amber-files/repository/targets.json
EOF
  # expect that "fx shell run-test-component URL-WITH-HASH" executes with the new hash, not the old one
  BT_EXPECT ${fx} test ${testname1} >> "${out}"
  local packageUrl="fuchsia-pkg://fuchsia.com/example-fuzzers?hash=${hash_new}#meta/overflow_fuzzer_test.cmx"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/shell" "run-test-component" "${packageUrl}"
}

# Test that "fx test" fails if a component test doesn't have an entry in the
# package repository
TEST_fxtest_no_hashfile() {
  cp -R "${DATA_DIR}/tests_hashfile/out" "${BT_TEMP_DIR}"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [],
    "test": {
      "cpu": "arm64",
      "label": "//examples/fuzzer:fuzzing-examples_pkg(//build/toolchain/fuchsia:arm64)",
      "name": "overflow_fuzzer_test",
      "os": "fuchsia",
      "package_url": "fuchsia-pkg://fuchsia.com/example_not_in_repository#meta/overflow_fuzzer_test.cmx",
      "path": ""
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"

  # expect that fx test fails because the repository doesn't have an entry for
  # this test's package (example_not_in_repository)
  BT_EXPECT_FAIL ${fx} test --no-build overflow_fuzzer_test >> "${out}"
}

# Test that "fx test" builds only the minimal target for device tests
TEST_fxtest_build_device_only_package() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [],
    "test": {
      "cpu": "arm64",
      "label": "//examples/fuzzer:fuzzing-examples_pkg(//build/toolchain/fuchsia:arm64)",
      "package_label": "//examples/fuzzer:fuzzing_pkg(//build/toolchain/fuchsia:arm64)",
      "name": "overflow_fuzzer_test",
      "os": "fuchsia",
      "package_url": "fuchsia-pkg://fuchsia.com/example-fuzzers#meta/overflow_fuzzer_test.cmx",
      "path": ""
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"

  # with incremental enabled, expect "fx shell build <test_package>"
  BT_EXPECT ${fx} --enable=incremental test --no-use-package-hash overflow_fuzzer_test >> "${out}"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/build" "examples/fuzzer:fuzzing_pkg"
  # with incremental disabled, expect "fx shell build updates"
  BT_EXPECT ${fx} --disable=incremental test --no-use-package-hash overflow_fuzzer_test >> "${out}"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/build.mock_state.2" "updates"
}

# Test that "fx test" builds the default target for an e2e test
TEST_fxtest_build_e2e() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [{"dimensions": {"device_type": "qemu_x64"}}],
    "test": {
      "cpu": "x64",
      "label": "//examples/example_host_test:host_tools_example_pkg(//build/toolchain/host_x64)",
      "name": "example_host_test",
      "os": "linux",
      "path": "host_x64/example_host_test"
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"

  btf::make_hosttools_mock "example_host_test" > /dev/null

  # expect "fx shell build"
  BT_EXPECT ${fx} test --e2e example_host_test >> "${out}"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/build"
}

# Test that "fx test" only builds the host tool for a host test
TEST_fxtest_build_host() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [],
    "test": {
      "cpu": "x64",
      "label": "//examples/example_host_test:host_tools_example_pkg(//build/toolchain/host_x64)",
      "name": "example_host_test",
      "os": "linux",
      "path": "host_x64/example_host_test"
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"

  btf::make_hosttools_mock "example_host_test" > /dev/null

  # expect "fx shell build"
  BT_EXPECT ${fx} test example_host_test >> "${out}"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/build" "host_x64/example_host_test"
}

# Test that "fx test" executes an e2e test with the proper env variables set
TEST_fxtest_e2e_env() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [{"dimensions": {"device_type": "qemu_x64"}}],
    "test": {
      "cpu": "x64",
      "label": "//examples/example_host_test:host_tools_example_pkg(//build/toolchain/host_x64)",
      "name": "example_host_test",
      "os": "linux",
      "path": "host_x64/example_host_test"
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local test_exec="$(btf::make_hosttools_mock "example_host_test")"
  # some 'fx test' features call this function to track test execution if the
  # user has metrics enabled.
  function track-subcommand-custom-event {
    :
  }
  export -f track-subcommand-custom-event

  # when the test is executed, print out the e2e variables as they are given to the test
  cat > "${test_exec}.mock_side_effects" <<'EOF'
  printVar() {
    local n="$1"
    # print var if it's set, even if it's empty
    [[ -n ${!n+x} ]] && echo "TESTING_E2E_VARS: $n=${!n}" > /dev/stderr
  }
  printVar FUCHSIA_DEVICE_ADDR
  printVar FUCHSIA_SSH_KEY
  printVar FUCHSIA_SSH_PORT
  printVar FUCHSIA_TEST_OUTDIR
  printVar SL4F_HTTP_PORT
  printVar FUCHSIA_IPV4_ADDR
  exit 0
EOF
  # check if environemnt is as expected for a user-defined IPV4 device and SSH port
  BT_EXPECT ${fx} -d 1.1.1.1:8123 test -o --e2e example_host_test | grep "TESTING_E2E_VARS" > "${out}"
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "FUCHSIA_DEVICE_ADDR=1.1.1.1"
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "FUCHSIA_SSH_PORT=8123"
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "FUCHSIA_SSH_KEY=..*"   # ensures no empty definition
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "FUCHSIA_TEST_OUTDIR=..*" # ensures no empty definition

  # check if environemnt is as expected for a user-defined IPV6 device and SSH port
  BT_EXPECT ${fx} -d '[::1]:1111' test -o --e2e example_host_test | grep "TESTING_E2E_VARS" > "${out}"
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "FUCHSIA_DEVICE_ADDR=[::1]"
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "FUCHSIA_SSH_PORT=1111"
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "FUCHSIA_SSH_KEY=..*"   # ensures no empty definition
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "FUCHSIA_TEST_OUTDIR=..*" # ensures no empty definition

  # no device set, test should not be given a fuchsia_ssh_port, and fuchsia_device_addr must be defined as empty
  BT_EXPECT ${fx} test -o --e2e example_host_test | grep "TESTING_E2E_VARS" > "${out}"
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" 'FUCHSIA_DEVICE_ADDR=$'  # ensures empty definition
  BT_EXPECT_EMPTY "$(grep FUCHSIA_SSH_PORT "${out}")" "unexpected FUCHSIA_SSH_PORT"

  # if defined in the caller's environment, sl4f_http_port is passed as is
  BT_EXPECT SL4F_HTTP_PORT=9099 ${fx} test -o --e2e example_host_test | grep "TESTING_E2E_VARS" > "${out}"
  BT_EXPECT_FILE_CONTAINS_SUBSTRING "${out}" "SL4F_HTTP_PORT=9099"

  # fuchsia_test_outdir must be a valid and writable directory
  BT_EXPECT ${fx} test -o --e2e example_host_test > "${out}"
  local test_out_dir="$(sed -n "s/.* FUCHSIA_TEST_OUTDIR=\(.*\)/\1/p" "${out}")"
  BT_EXPECT touch "${test_out_dir}/test_file"
}

# Ensure that if `fx build` changes `tests.json`, `fx test` loads the updated
# tests.json before running the test.
TEST_fxtest_build_changes_testsjson() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local test_exec="$(btf::make_hosttools_mock "example_host_test")"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
      [{"environments": [],
    "test": {
      "command": [
        "host_x64/example_host_test",
        "beforefxbuild"
      ],
      "cpu": "x64",
      "label": "//examples/example_host_test:host_tools_example_pkg(//build/toolchain/host_x64)",
      "name": "example_host_test",
      "os": "linux",
      "path": "host_x64/example_host_test"
    }
  }]
EOF
  # make 'fx build' change the command argument in tests.json from "beforefxbuild"
  # to "afterfxbuild"
  cat > "${BT_TEMP_DIR}/tools/devshell/build.mock_side_effects" <<EOF
    sed -i 's/beforefxbuild/afterfxbuild/' "${BT_TEMP_DIR}/out/default/tests.json"
EOF

  BT_EXPECT ${fx} test example_host_test > ${out}

  btf::expect-mock-args "${test_exec}.mock_state" "afterfxbuild"
}



BT_RUN_TESTS "$@"