| // Copyright 2018 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| // Helper functions for running test binaries and recording their results. |
| |
| #ifndef RUNTESTS_UTILS_RUNTESTS_UTILS_H_ |
| #define RUNTESTS_UTILS_RUNTESTS_UTILS_H_ |
| |
| #include <inttypes.h> |
| |
| #include <fbl/intrusive_hash_table.h> |
| #include <fbl/intrusive_single_list.h> |
| #include <fbl/string.h> |
| #include <fbl/string_piece.h> |
| #include <fbl/unique_ptr.h> |
| #include <fbl/vector.h> |
| #include <lib/zircon-internal/fnv1hash.h> |
| #include <zircon/types.h> |
| |
| namespace runtests { |
| |
| // Status of launching a test subprocess. |
| enum LaunchStatus { |
| SUCCESS, |
| FAILED_TO_LAUNCH, |
| FAILED_TO_WAIT, |
| FAILED_DURING_IO, |
| FAILED_TO_RETURN_CODE, |
| FAILED_NONZERO_RETURN_CODE, |
| FAILED_COLLECTING_SINK_DATA, |
| FAILED_UNKNOWN, |
| }; |
| |
| // Represents a single dumpfile element. |
| struct DumpFile { |
| fbl::String name; // Name of the dumpfile. |
| fbl::String file; // File name for the content. |
| }; |
| |
| // Represents data published through data sink. |
| struct DataSink : public fbl::SinglyLinkedListable<std::unique_ptr<DataSink>> { |
| fbl::String name; // Name of the data sink. |
| fbl::Vector<DumpFile> files; // All the sink dumpfiles. |
| |
| explicit DataSink(fbl::String name) |
| : name(name) {} |
| |
| // Runtimes may publish more than one file with the same data sink name. We use hash table |
| // that's mapping data sink name to the list of file names to store these efficiently. |
| fbl::String GetKey() const { return name; } |
| static size_t GetHash(fbl::String key) { return fnv1a64str(key.c_str()); } |
| }; |
| |
| // Represents the result of a single test run. |
| struct Result { |
| fbl::String name; // argv[0]. |
| LaunchStatus launch_status; |
| int64_t return_code; // Only valid if launch_status == SUCCESS or FAILED_NONZERO_RETURN_CODE. |
| using HashTable = fbl::HashTable<fbl::String, std::unique_ptr<DataSink>>; |
| HashTable data_sinks; // Mapping from data sink name to list of files. |
| // TODO(ZX-2050): Track duration of test binary. |
| |
| // Constructor really only needed until we have C++14, which will allow call-sites to use |
| // aggregate initializer syntax. |
| Result(const char* name_arg, LaunchStatus launch_status_arg, int64_t return_code_arg) |
| : name(name_arg), launch_status(launch_status_arg), return_code(return_code_arg) {} |
| }; |
| |
| // Function that invokes a test binary and writes its output to a file. |
| // |
| // |argv| is the commandline to use to run the test program; must be |
| // null-terminated. |
| // |output_dir| is the output directory for test's data sinks. May be nullptr, in which case |
| // no data sinks will be saved. |
| // |output_filename| is the name of the file to which the test binary's output |
| // will be written. May be nullptr, in which case the output will not be |
| // redirected. |
| typedef std::unique_ptr<Result> (*RunTestFn)(const char* argv[], |
| const char* output_dir, |
| const char* output_filename); |
| |
| // A means of measuring how long it takes to run tests. |
| class Stopwatch { |
| public: |
| virtual ~Stopwatch() = default; |
| |
| // Starts timing. |
| virtual void Start() = 0; |
| |
| // Returns the elapsed time in milliseconds since invoking Start(), or else |
| // since initialization if Start() has not yet been called. |
| virtual int64_t DurationInMsecs() = 0; |
| }; |
| |
| // Splits |input| by ',' and appends the results onto |output|. |
| // Empty strings are not put into output. |
| void ParseTestNames(fbl::StringPiece input, fbl::Vector<fbl::String>* output); |
| |
| // Returns true iff |name| is equal to one of strings in |whitelist|. |
| bool IsInWhitelist(fbl::StringPiece name, const fbl::Vector<fbl::String>& whitelist); |
| |
| // Ensures |dir_name| exists by creating it and its parents if it doesn't. |
| // Returns 0 on success, else an error code compatible with errno. |
| int MkDirAll(fbl::StringPiece dir_name); |
| |
| // Returns "|parent|/|child|". Unless child is absolute, in which case it returns |child|. |
| // |
| // |parent| is the parent path. |
| // |child| is the child path. |
| fbl::String JoinPath(fbl::StringPiece parent, fbl::StringPiece child); |
| |
| // Writes a JSON summary of test results given a sequence of results. |
| // |
| // |results| are the run results to summarize. |
| // |output_file_basename| is base name of output file. |
| // |syslog_path| is the file path where syslogs are written. |
| // |summary_json| is the file stream to write the JSON summary to. |
| // |
| // Returns 0 on success, else an error code compatible with errno. |
| int WriteSummaryJSON(const fbl::Vector<std::unique_ptr<Result>>& results, |
| fbl::StringPiece output_file_basename, |
| fbl::StringPiece syslog_path, |
| FILE* summary_json); |
| |
| // Resolves a set of globs. |
| // |
| // |globs| is an array of glob patterns. |
| // |resolved| will hold the results of resolving |globs|. |
| // |
| // Returns 0 on success, else an error code from glob.h. |
| int ResolveGlobs(const fbl::Vector<fbl::String>& globs, |
| fbl::Vector<fbl::String>* resolved); |
| |
| // Executes all specified binaries. |
| // |
| // |run_test| is the function used to invoke the test binaries. |
| // |test_paths| are the paths of the binaries to execute. |
| // |test_args| are arguments passed into the binaries under test. |
| // |repeat| runs the entire test suite this many times. The entire suite is repeated rather than |
| // each test individually so that: |
| // a) any flakes due to the sequencing of tests can be reproduced |
| // b) we can get an idea of global flake rates without waiting for all runs to complete |
| // |output_dir| is the output directory for all the tests' output. May be nullptr, in which case |
| // output will not be captured. |
| // |output_file_basename| is the basename of the tests' output files. May be nullptr only if |
| // |output_dir| is also nullptr. |
| // Each test's standard output and standard error will be written to |
| // |output_dir|/<test binary path>/|output_file_basename|. |
| // |verbosity| if > 0 is converted to a string and passed as an additional argument to the |
| // tests, so argv = {test_path, "v=<verbosity>"}. Also if > 0, this function prints more output |
| // to stdout than otherwise. |
| // |num_failed| is an output parameter which will be set to the number of test |
| // binaries that failed. |
| // |results| is an output parameter to which run results will be appended. |
| // |
| // Returns false if any test binary failed, true otherwise. |
| bool RunTests(const RunTestFn& RunTest, const fbl::Vector<fbl::String>& test_paths, |
| const fbl::Vector<fbl::String>& test_args, int repeat, |
| const char* output_dir, const fbl::StringPiece output_file_basename, |
| signed char verbosity, int* failed_count, |
| fbl::Vector<std::unique_ptr<Result>>* results); |
| |
| // Expands |dir_globs| and searches those directories for files. |
| // |
| // |dir_globs| are expanded as globs to directory names, and then those directories are searched. |
| // |ignore_dir_name| iff not null, any directory with this basename will not be searched. |
| // |basename_whitelist| iff not empty, only files that have a basename in this whitelist will be |
| // returned. |
| // |test_paths| is an output parameter to which absolute file paths will be appended. |
| // |
| // Returns 0 on success, else an error code compatible with errno. |
| int DiscoverTestsInDirGlobs(const fbl::Vector<fbl::String>& dir_globs, const char* ignore_dir_name, |
| const fbl::Vector<fbl::String>& basename_whitelist, |
| fbl::Vector<fbl::String>* test_paths); |
| |
| // Reads |test_list_file| and appends whatever tests it finds to |test_paths|. |
| // |
| // Returns 0 on success, else an error code compatible with errno. |
| int DiscoverTestsInListFile(FILE* test_list_file, fbl::Vector<fbl::String>* test_paths); |
| |
| // Discovers and runs tests based on command line arguments. |
| // |
| // |RunTest|: function to run each test. |
| // |argc|: length of |argv|. |
| // |argv|: see //system/ulib/runtests-utils/discover-and-run-tests.cpp, |
| // specifically the 'Usage()' function, for documentation. |
| // |default_test_dirs|: directories in which to look for tests if no test |
| // directory globs are specified. |
| // |stopwatch|: for timing how long all tests took to run. |
| // |syslog_file_name|: if an output directory is specified ("-o"), syslog ouput |
| // will be written to a file under that directory and this name. |
| // |
| // Returns EXIT_SUCCESS if all tests passed; else, returns EXIT_FAILURE. |
| int DiscoverAndRunTests(const RunTestFn& RunTest, int argc, const char* const* argv, |
| const fbl::Vector<fbl::String>& default_test_dirs, |
| Stopwatch* stopwatch, const fbl::StringPiece syslog_file_name); |
| |
| } // namespace runtests |
| |
| #endif // RUNTESTS_UTILS_RUNTESTS_UTILS_H_ |