| // Copyright 2018 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include <runtests-utils/runtests-utils.h> |
| |
| #include <errno.h> |
| #include <fcntl.h> |
| #include <inttypes.h> |
| #include <libgen.h> |
| #include <limits.h> |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <sys/stat.h> |
| #include <unistd.h> |
| |
| #include <fbl/string.h> |
| #include <fbl/vector.h> |
| #include <unittest/unittest.h> |
| |
| #include <utility> |
| |
| namespace runtests { |
| namespace { |
| |
| // The name of the file containing stdout and stderr of a test. |
| constexpr char kOutputFileName[] = "stdout-and-stderr.txt"; |
| |
| // Ignore test directories where the last component is this. This permits users |
| // to specify a more general glob that might match to a subdirectory |
| // containing data for a particular test, which would result in failure should |
| // runtests try to enter it. |
| constexpr char kIgnoreDirName[] = "helper"; |
| |
| int Usage(const char* name, const fbl::Vector<fbl::String>& default_test_dirs) { |
| bool test_dirs_required = default_test_dirs.is_empty(); |
| fprintf(stderr, |
| "Usage: %s [-q|-v] [-S|-s] [-M|-m] [-L|-l] [-P|-p] [-a]\n" |
| " [-w timeout] [-t test names] [-o directory] \n" |
| " [directory globs ...] \n" |
| " [-- [-args -to -the -test -bin]] \n" |
| "\n" |
| "The %s [directory globs...] is a list of \n" |
| "globs which match directories containing tests to run,\n" |
| "non-recursively. Note that non-directories captured by\n" |
| "a glob will be silently ignored. \n" |
| "\n" |
| "After directory globs, `--` can be followed by any \n" |
| "number of arguments to pass to the binaries under \n" |
| "test.\n", |
| name, test_dirs_required ? "required" : "optional"); |
| if (!test_dirs_required) { |
| fprintf(stderr, |
| "If unspecified, the default set of directories is\n"); |
| for (const auto& test_dir : default_test_dirs) { |
| fprintf(stderr, " %s\n", test_dir.c_str()); |
| } |
| } |
| fprintf(stderr, |
| "\noptions: \n" |
| " -h: See this message \n" |
| " -d: Dry run, just print test file names and exit \n" |
| " -v: Verbose output [1] \n" |
| " -q: Quiet output \n" |
| " -S: Turn ON Small tests (on by default) [2] \n" |
| " -s: Turn OFF Small tests [2] \n" |
| " -M: Turn ON Medium tests (on by default) [2] \n" |
| " -m: Turn OFF Medium tests [2] \n" |
| " -L: Turn ON Large tests (off by default) [2] \n" |
| " -l: Turn OFF Large tests [2] \n" |
| " -P: Turn ON Performance tests (off by default) [2] \n" |
| " -p: Turn OFF Performance tests [2] \n" |
| " -a: Turn on All tests \n" |
| " -t: Filter tests by name \n" |
| " (accepts a comma-separated list) \n" |
| " -f: Run tests specified in this file [3] \n" |
| " -r: Repeat the test suite this many times \n" |
| " -o: Write test output to a directory [4] \n" |
| " -w: Watchdog timeout [5] \n" |
| " (accepts the timeout value in seconds) \n" |
| " The default is up to each test. \n" |
| "\n" |
| "[1] -v will pass \"v=1\" argument to the test binary. \n" |
| " Not all test frameworks will honor this argument, \n" |
| " and some, like Rust, may interpret it as a filter.\n" |
| " Use \"-- --nocapture\" for a similar behaviour \n" |
| " when running Rust tests. \n" |
| "\n" |
| "[2] The test selection options -[sSmMlLpP] only work \n" |
| " for tests that support the RUNTESTS_TEST_CLASS \n" |
| " environment variable. \n" |
| "\n" |
| "[3] -f and [directory globs ...] are mutually \n" |
| " exclusive. \n" |
| "\n" |
| "[4] If -o is enabled, then a JSON summary of the test \n" |
| " results will be written to a file named \n" |
| " \"summary.json\" under the desired directory, in \n" |
| " addition to each test's standard output and error.\n" |
| " The summary contains a listing of the tests \n" |
| " executed by full path (e.g., \n" |
| " /boot/test/core/futex_test), as well as whether \n" |
| " the test passed or failed. For details, see \n" |
| " //system/ulib/runtests-utils/summary-schema.json \n" |
| "\n" |
| "[5] The watchdog timeout option -w only works for \n" |
| " tests that support the RUNTESTS_WATCHDOG_TIMEOUT \n" |
| " environment variable. \n"); |
| return EXIT_FAILURE; |
| } |
| } // namespace |
| |
| int DiscoverAndRunTests(const RunTestFn& RunTest, int argc, const char* const* argv, |
| const fbl::Vector<fbl::String>& default_test_dirs, Stopwatch* stopwatch, |
| const fbl::StringPiece syslog_file_name) { |
| unsigned int test_types = TEST_DEFAULT; |
| fbl::Vector<fbl::String> basename_whitelist; |
| fbl::Vector<fbl::String> test_dir_globs; |
| fbl::Vector<fbl::String> test_args; |
| const char* output_dir = nullptr; |
| signed char verbosity = -1; |
| int watchdog_timeout_seconds = -1; |
| const char* test_list_path = nullptr; |
| int repeat = 1; |
| bool dry_run = false; |
| |
| int optind = 1; |
| while (optind < argc) { |
| // Implementing our own opt parsing here is less effort that fixing up |
| // the behavior across three different getopt implementations on macos, |
| // linux and zircon, even with this comment. The breaking requirement is |
| // to parse globs at any position in argv. |
| fbl::String arg(argv[optind++]); |
| |
| if (arg.length() == 0) { |
| continue; |
| } |
| |
| if (arg == "--") { |
| for (; optind < argc; ++optind) { |
| test_args.push_back(argv[optind]); |
| } |
| break; |
| } |
| |
| if (arg.length() < 2 || arg.data()[0] != '-') { |
| test_dir_globs.push_back(std::move(arg)); |
| continue; |
| } |
| |
| switch (arg.data()[1]) { |
| case 'q': |
| verbosity = 0; |
| break; |
| case 'v': |
| fprintf(stderr, "verbose output. enjoy.\n"); |
| verbosity = 1; |
| break; |
| case 's': |
| test_types &= ~TEST_SMALL; |
| break; |
| case 'm': |
| test_types &= ~TEST_MEDIUM; |
| break; |
| case 'l': |
| test_types &= ~TEST_LARGE; |
| break; |
| case 'p': |
| test_types &= ~TEST_PERFORMANCE; |
| break; |
| case 'S': |
| test_types |= TEST_SMALL; |
| break; |
| case 'M': |
| test_types |= TEST_MEDIUM; |
| break; |
| case 'L': |
| test_types |= TEST_LARGE; |
| break; |
| case 'P': |
| test_types |= TEST_PERFORMANCE; |
| break; |
| case 'a': |
| test_types |= TEST_ALL; |
| break; |
| case 'h': |
| return Usage(argv[0], default_test_dirs); |
| case 't': |
| if (optind > argc) { |
| fprintf(stderr, "Missing argument for -t\n"); |
| return EXIT_FAILURE; |
| } |
| ParseTestNames(argv[optind++], &basename_whitelist); |
| break; |
| case 'o': |
| if (optind > argc) { |
| fprintf(stderr, "Missing argument for -o\n"); |
| return EXIT_FAILURE; |
| } |
| output_dir = argv[optind++]; |
| break; |
| case 'f': |
| if (optind > argc) { |
| fprintf(stderr, "Missing argument for -f\n"); |
| return EXIT_FAILURE; |
| } |
| test_list_path = argv[optind++]; |
| break; |
| case 'r': { |
| if (optind > argc) { |
| fprintf(stderr, "Missing argument for -r\n"); |
| return EXIT_FAILURE; |
| } |
| const char* repeat_str = argv[optind++]; |
| char* end; |
| long repeatl = strtol(repeat_str, &end, 0); |
| if (*repeat_str == '\0' || *end != '\0' || repeatl < 0 || repeatl > INT_MAX) { |
| fprintf(stderr, "Error: bad repeat\n"); |
| return EXIT_FAILURE; |
| } |
| repeat = static_cast<int>(repeatl); |
| break; |
| } |
| case 'w': { |
| if (optind > argc) { |
| fprintf(stderr, "Missing argument for -w\n"); |
| return EXIT_FAILURE; |
| } |
| const char* timeout_str = argv[optind++]; |
| char* end; |
| long timeout = strtol(timeout_str, &end, 0); |
| if (*timeout_str == '\0' || *end != '\0' || timeout < 0 || timeout > INT_MAX) { |
| fprintf(stderr, "Error: bad timeout\n"); |
| return EXIT_FAILURE; |
| } |
| watchdog_timeout_seconds = static_cast<int>(timeout); |
| break; |
| } |
| case 'd': { |
| dry_run = true; |
| break; |
| } |
| default: |
| return Usage(argv[0], default_test_dirs); |
| } |
| } |
| |
| if (test_list_path && !test_dir_globs.is_empty()) { |
| fprintf(stderr, "Can't set both -f and directory globs.\n"); |
| return Usage(argv[0], default_test_dirs); |
| } |
| |
| // Configure the types of tests which are meant to be executed by putting |
| // it in an environment variable. Test executables can consume this environment |
| // variable and process it as they would like. |
| char test_opt[32]; |
| snprintf(test_opt, sizeof(test_opt), "%u", test_types); |
| if (setenv(TEST_ENV_NAME, test_opt, 1) != 0) { |
| fprintf(stderr, "Error: Could not set %s environment variable\n", TEST_ENV_NAME); |
| return EXIT_FAILURE; |
| } |
| |
| // If set, configure the watchdog timeout to use. |
| if (watchdog_timeout_seconds >= 0) { |
| char timeout_str[32]; |
| snprintf(timeout_str, sizeof(timeout_str), "%d", watchdog_timeout_seconds); |
| if (setenv(WATCHDOG_ENV_NAME, timeout_str, 1) != 0) { |
| fprintf(stderr, "Error: Could not set %s environment variable\n", WATCHDOG_ENV_NAME); |
| return EXIT_FAILURE; |
| } |
| } else { |
| // Ensure we don't pass on any existing value. This is intentional: |
| // If -w is not specified then that means the watchdog is unspecified, |
| // period. |
| unsetenv(WATCHDOG_ENV_NAME); |
| } |
| |
| fbl::Vector<fbl::String> test_paths; |
| const auto* test_dir_globs_or_default = |
| test_dir_globs.is_empty() ? &default_test_dirs : &test_dir_globs; |
| if (test_list_path) { |
| FILE* test_list_file = fopen(test_list_path, "r"); |
| if (!test_list_file) { |
| fprintf(stderr, "Failed to open test list file %s: %s\n", test_list_path, |
| strerror(errno)); |
| return false; |
| } |
| const int err = DiscoverTestsInListFile(test_list_file, &test_paths); |
| fclose(test_list_file); |
| if (err) { |
| fprintf(stderr, "Failed to read test list from %s: %s\n", test_list_path, |
| strerror(err)); |
| return EXIT_FAILURE; |
| } |
| } else if (!test_dir_globs_or_default->is_empty()) { |
| const int err = DiscoverTestsInDirGlobs(*test_dir_globs_or_default, kIgnoreDirName, |
| basename_whitelist, &test_paths); |
| if (err) { |
| fprintf(stderr, "Failed to find tests in dirs: %s\n", strerror(err)); |
| return EXIT_FAILURE; |
| } |
| } else { |
| fprintf( |
| stderr, |
| "Test list path, test directory globs or default test directories must be specified."); |
| return EXIT_FAILURE; |
| } |
| |
| struct stat st; |
| if (output_dir != nullptr && stat(output_dir, &st) < 0 && (st.st_mode & S_IFMT) == S_IFDIR) { |
| fprintf(stderr, "Error: Could not open %s\n", output_dir); |
| return EXIT_FAILURE; |
| } |
| |
| if (dry_run) { |
| printf("Would run the following tests:\n"); |
| for (const auto& test_path : test_paths) { |
| printf("\t%s\n", test_path.c_str()); |
| } |
| return EXIT_SUCCESS; |
| } |
| |
| // TODO(mknyszek): Sort test_paths for deterministic behavior. Should be easy after ZX-1751. |
| stopwatch->Start(); |
| int failed_count = 0; |
| fbl::Vector<std::unique_ptr<Result>> results; |
| if (!RunTests(RunTest, test_paths, test_args, repeat, output_dir, kOutputFileName, verbosity, |
| &failed_count, &results)) { |
| return EXIT_FAILURE; |
| } |
| |
| // It's not catastrophic if we can't unset it; we're just trying to clean up |
| unsetenv(TEST_ENV_NAME); |
| unsetenv(WATCHDOG_ENV_NAME); |
| |
| if (output_dir != nullptr) { |
| char summary_path[PATH_MAX]; |
| snprintf(summary_path, sizeof(summary_path), "%s/summary.json", output_dir); |
| FILE* summary_json = fopen(summary_path, "w"); |
| if (summary_json == nullptr) { |
| fprintf(stderr, "Error: Could not open JSON summary file.\n"); |
| return EXIT_FAILURE; |
| } |
| const int error = WriteSummaryJSON(results, kOutputFileName, |
| syslog_file_name, summary_json); |
| if (error) { |
| fprintf(stderr, "Error: Failed to write JSON summary: %s\n", strerror(error)); |
| return EXIT_FAILURE; |
| } |
| if (fclose(summary_json)) { |
| fprintf(stderr, "Error: Could not close JSON summary.\n"); |
| return EXIT_FAILURE; |
| } |
| } |
| |
| // Display any failed tests, and free the test results. |
| if (failed_count) { |
| printf("\nThe following tests failed:\n"); |
| } |
| for (const std::unique_ptr<Result>& result : results) { |
| switch (result->launch_status) { |
| case SUCCESS: |
| break; |
| case FAILED_TO_LAUNCH: |
| printf("%s: failed to launch\n", result->name.c_str()); |
| break; |
| case FAILED_TO_WAIT: |
| printf("%s: failed to wait\n", result->name.c_str()); |
| break; |
| case FAILED_TO_RETURN_CODE: |
| printf("%s: failed to return exit code\n", result->name.c_str()); |
| break; |
| case FAILED_NONZERO_RETURN_CODE: |
| printf("%s: returned nonzero: %" PRId64 "\n", |
| result->name.c_str(), result->return_code); |
| break; |
| default: |
| printf("%s: unknown result\n", result->name.c_str()); |
| break; |
| } |
| } |
| |
| // TODO(ZX-2051): Include total duration in summary.json. |
| uint64_t time_taken_ms = stopwatch->DurationInMsecs(); |
| |
| // Print this last, since some infra recipes will shut down the fuchsia |
| // environment once it appears. |
| printf("\nSUMMARY: Ran %lu tests: %d failed (%" PRIu64 ".%03u sec)\n", results.size(), |
| failed_count, time_taken_ms / 1000, (unsigned)(time_taken_ms % 1000)); |
| |
| return failed_count ? EXIT_FAILURE : EXIT_SUCCESS; |
| } |
| |
| } // namespace runtests |
