tools/devshell/tests/subcommands/fx_test_test

#!/bin/bash
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

### Test expected behavior of fx test

BT_LINKED_DEPS=(
  "third_party"
  "prebuilt/third_party/dart"
  "scripts/fxtest"
)

BT_FILE_DEPS=(
  "scripts/fx"
  "tools/devshell/tests/subcommands/data/fx_test_test/tests_hashfile"
  "tools/devshell/tests/subcommands/data/fx_test_test/tests_multiple_in_package.json"
  "tools/devshell/tests/subcommands/data/fx_test_test/tests_package_server_integration.json"
  "tools/devshell/lib/fx-cmd-locator.sh"
  "tools/devshell/lib/fx-optional-features.sh"
  "tools/devshell/lib/vars.sh"
  "tools/devshell/lib/prebuilt.sh"
  "tools/devshell/test"
)

BT_MOCKED_TOOLS=(
  "tools/devshell/build"
  "tools/devshell/is-package-server-running"
  "tools/devshell/update-if-in-base"
  "tools/devshell/shell"
  "tools/devshell/symbolize"
)

declare fx DATA_DIR

BT_SET_UP() {
  source "${BT_TEMP_DIR}/tools/devshell/tests/lib/fuchsia-mock.sh"
  fx="$(btf::setup_fx)"
  btf::make_installed_hosttools_mock "device-finder" > /dev/null
  DATA_DIR="${BT_TEMP_DIR}/tools/devshell/tests/subcommands/data/fx_test_test"
}

# Test that the "fx test --info" outputs in the format expected by other
# commands, eg `fx run-test`
TEST_fxtest_info() {
  cp "${DATA_DIR}/tests_multiple_in_package.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  BT_EXPECT ${fx} test --info --exact > "${out}"
  BT_EXPECT_EQ "$(sed -n 's/^package_url: \(.*\)/\1/p' "${out}" | wc -l)" 7
}

# Test that `fx test` calls `fx update-if-in-base` and `fx is-package-server-running` properly
TEST_fxtest_package_server_integration() {
  cp "${DATA_DIR}/tests_package_server_integration.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname="overflow_fuzzer_test"
  BT_EXPECT ${fx} test ${testname} > ${out}
  # ensure that is-package-server-running was called
  BT_ASSERT_FILE_EXISTS "${BT_TEMP_DIR}/tools/devshell/is-package-server-running.mock_state"
  # ensure that update-if-in-base was called with the proper testname
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/update-if-in-base" "${testname}"
}

# Ensure that `fx build` is called by default
TEST_fxtest_build() {
  cp "${DATA_DIR}/tests_package_server_integration.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname="overflow_fuzzer_test"
  BT_EXPECT ${fx} test ${testname} > ${out}
  # ensure that fx build was called
  # TODO: once fx test calls fx build with a specific target, check it here as well
  BT_ASSERT_FILE_EXISTS "${BT_TEMP_DIR}/tools/devshell/build.mock_state"
}

# Ensure that `fx build` is not called when "--no-build" option is given
TEST_fxtest_nobuild() {
  cp "${DATA_DIR}/tests_package_server_integration.json" "${BT_TEMP_DIR}/out/default/tests.json"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname="overflow_fuzzer_test"
  BT_EXPECT ${fx} test --no-build ${testname} > ${out}
  # ensure that fx build was called
  # TODO: once fx test calls fx build with a specific target, check it here as well
  BT_ASSERT_FILE_DOES_NOT_EXIST "${BT_TEMP_DIR}/tools/devshell/build.mock_state"
}

# Test that "fx test" runs a component test pinning it to the hash (merkleroot) of
# the component package, so that the user has confidence that if a test runs, it is
# running the exact same version that has been built
TEST_fxtest_hashpinnning() {
  cp -R "${DATA_DIR}/tests_hashfile/out" "${BT_TEMP_DIR}"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname1="overflow_fuzzer_test"
  local hash1="913cdd63ab4aa794694448450505efaa2a8fe27fb33888e5156da9db60ac0a29"
  local testname2="hello_world_cpp_unittests"
  local hash2="7a604498e05fa012391b6b51da9cc74ff6a6a9d25b1376de98125c194232bfa1"

  # expect that "fx shell run-test-component URL-WITH-HASH" was executed
  BT_EXPECT ${fx} test --no-build ${testname1} >> "${out}"
  local packageUrl1="fuchsia-pkg://fuchsia.com/example-fuzzers?hash=${hash1}#meta/overflow_fuzzer_test.cmx"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/shell" "run-test-component" "_ANY_" "${packageUrl1}"

  # expect that "fx shell run-test-component URL-WITH-HASH" was executed
  BT_EXPECT ${fx} test --no-build ${testname2} >> "${out}"
  local packageUrl2="fuchsia-pkg://fuchsia.com/hello_world_cpp_tests?hash=${hash2}#meta/hello_world_cpp_unittests.cmx"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/shell.mock_state.2" "run-test-component" "_ANY_" "${packageUrl2}"
}

# Test that when the merkle root of a package is changed as a result of the
# 'fx build' executed by fx test, the new hash is used when pinning the
# package in fx shell run-test-component
TEST_fxtest_updated_hash() {
  cp -R "${DATA_DIR}/tests_hashfile/out" "${BT_TEMP_DIR}"
  local out="${BT_TEMP_DIR}/_fx_test_output"
  local testname1="overflow_fuzzer_test"
  local hash_old="913cdd63ab4aa794694448450505efaa2a8fe27fb33888e5156da9db60ac0a29"
  local hash_new="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"

  # create a build script that, when called, changes hash_old to hash_new in the
  # package repository
  cat > "${BT_TEMP_DIR}/tools/devshell/build.mock_side_effects" <<EOF
  sed -i 's/${hash_old}/${hash_new}/g' ${BT_TEMP_DIR}/out/default/amber-files/repository/targets.json
EOF
  # expect that "fx shell run-test-component URL-WITH-HASH" executes with the new hash, not the old one
  BT_EXPECT ${fx} test ${testname1} >> "${out}"
  local packageUrl="fuchsia-pkg://fuchsia.com/example-fuzzers?hash=${hash_new}#meta/overflow_fuzzer_test.cmx"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/shell" "run-test-component" "_ANY_" "${packageUrl}"
}

# Test that "fx test" doesn't fail if a component test doesn't have a hashfile
# property
TEST_fxtest_no_hashfile() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [],
    "test": {
      "cpu": "arm64",
      "label": "//examples/fuzzer:fuzzing-examples_pkg(//build/toolchain/fuchsia:arm64)",
      "name": "overflow_fuzzer_test",
      "os": "fuchsia",
      "package_url": "fuchsia-pkg://fuchsia.com/example-fuzzers#meta/overflow_fuzzer_test.cmx",
      "path": ""
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"

  # expect that "fx shell run-test-component URL-WITHOUT-HASH" is executed
  BT_EXPECT ${fx} test --no-build overflow_fuzzer_test >> "${out}"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/shell" \
    "run-test-component" "_ANY_" "fuchsia-pkg://fuchsia.com/example-fuzzers#meta/overflow_fuzzer_test.cmx"
}

# Test that "fx test" builds only the "updates" target for device tests
TEST_fxtest_build_device_only_updates() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [],
    "test": {
      "cpu": "arm64",
      "label": "//examples/fuzzer:fuzzing-examples_pkg(//build/toolchain/fuchsia:arm64)",
      "name": "overflow_fuzzer_test",
      "os": "fuchsia",
      "package_url": "fuchsia-pkg://fuchsia.com/example-fuzzers#meta/overflow_fuzzer_test.cmx",
      "path": ""
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"

  # expect "fx shell build updates"
  BT_EXPECT ${fx} test overflow_fuzzer_test >> "${out}"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/build" "updates"
}

# Test that "fx test" builds the default target for an e2e test
TEST_fxtest_build_e2e() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [{"dimensions": {"device_type": "qemu_x64"}}],
    "test": {
      "cpu": "x64",
      "label": "//examples/example_host_test:host_tools_example_pkg(//build/toolchain/host_x64)",
      "name": "example_host_test",
      "os": "linux",
      "path": "host_x64/example_host_test"
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"

  btf::make_hosttools_mock "example_host_test" > /dev/null

  # expect "fx shell build"
  BT_EXPECT ${fx} test --e2e example_host_test >> "${out}"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/build"
}

# Test that "fx test" only builds the host tool for a host test
TEST_fxtest_build_host() {
  mkdir -p "${BT_TEMP_DIR}/out/default"
  cat > "${BT_TEMP_DIR}/out/default/tests.json" <<EOF
  [{"environments": [],
    "test": {
      "cpu": "x64",
      "label": "//examples/example_host_test:host_tools_example_pkg(//build/toolchain/host_x64)",
      "name": "example_host_test",
      "os": "linux",
      "path": "host_x64/example_host_test"
    }
  }]
EOF
  local out="${BT_TEMP_DIR}/_fx_test_output"

  btf::make_hosttools_mock "example_host_test" > /dev/null

  # expect "fx shell build"
  BT_EXPECT ${fx} test example_host_test >> "${out}"
  btf::expect-mock-args "${BT_TEMP_DIR}/tools/devshell/build" "host_x64/example_host_test"
}

BT_RUN_TESTS "$@"