blob: 24bedcdeeadb7f56a7c5f5553cfd23ae8f4835b3 [file] [log] [blame] [edit]
// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <ctype.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <glob.h>
#include <inttypes.h>
#include <libgen.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <zircon/status.h>
#include <zircon/syscalls.h>
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <fbl/auto_call.h>
#include <fbl/string.h>
#include <fbl/string_buffer.h>
#include <fbl/string_piece.h>
#include <fbl/string_printf.h>
#include <fbl/vector.h>
#include <runtests-utils/runtests-utils.h>
namespace runtests {
// Signatures to print to indicate failure or success.
// Testrunner looks for these exact strings, follows by the test name to determine test status.
constexpr char kFailureSignature[] = "\n[runtests][FAILED]";
constexpr char kSuccessSignature[] = "\n[runtests][PASSED]";
void ParseTestNames(const fbl::StringPiece input, fbl::Vector<fbl::String>* output) {
// strsep modifies its input, so we have to make a mutable copy.
// +1 because StringPiece::size() excludes null terminator.
std::unique_ptr<char[]> input_copy(new char[input.size() + 1]);
memcpy(input_copy.get(), input.data(), input.size());
input_copy[input.size()] = '\0';
// Tokenize the input string into names.
char* next_token;
for (char* tmp = strtok_r(input_copy.get(), ",", &next_token); tmp != nullptr;
tmp = strtok_r(nullptr, ",", &next_token)) {
output->push_back(fbl::String(tmp));
}
}
bool IsInWhitelist(const fbl::StringPiece name, const fbl::Vector<fbl::String>& whitelist) {
for (const fbl::String& whitelist_entry : whitelist) {
if (name == fbl::StringPiece(whitelist_entry)) {
return true;
}
}
return false;
}
int MkDirAll(const fbl::StringPiece dir_name) {
fbl::StringBuffer<PATH_MAX> dir_buf;
if (dir_name.length() > dir_buf.capacity()) {
return ENAMETOOLONG;
}
dir_buf.Append(dir_name);
char* dir = dir_buf.data();
// Fast path: check if the directory already exists.
struct stat s;
if (!stat(dir, &s)) {
return 0;
}
// Slow path: create the directory and its parents.
for (size_t slash = 0u; dir[slash]; slash++) {
if (slash != 0u && dir[slash] == '/') {
dir[slash] = '\0';
if (mkdir(dir, 0755) && errno != EEXIST) {
return false;
}
dir[slash] = '/';
}
}
if (mkdir(dir, 0755) && errno != EEXIST) {
return errno;
}
return 0;
}
fbl::String JoinPath(const fbl::StringPiece parent, const fbl::StringPiece child) {
if (parent.empty()) {
return fbl::String(child);
}
if (child.empty()) {
return fbl::String(parent);
}
if (parent[parent.size() - 1] != '/' && child[0] != '/') {
return fbl::String::Concat({parent, "/", child});
}
if (parent[parent.size() - 1] == '/' && child[0] == '/') {
return fbl::String::Concat({parent, &child[1]});
}
return fbl::String::Concat({parent, child});
}
int WriteSummaryJSON(const fbl::Vector<std::unique_ptr<Result>>& results,
const fbl::StringPiece output_file_basename,
const fbl::StringPiece syslog_path, FILE* summary_json) {
int test_count = 0;
fprintf(summary_json, "{\n \"tests\": [\n");
for (const auto& result : results) {
if (test_count != 0) {
fprintf(summary_json, ",\n");
}
fprintf(summary_json, " {\n");
// Write the name of the test.
fprintf(summary_json, " \"name\": \"%s\",\n", result->name.c_str());
// Write the path to the output file, relative to the test output root
// (i.e. what's passed in via -o). The test name is already a path to
// the test binary on the target, so to make this a relative path, we
// only have to skip leading '/' characters in the test name.
fbl::String output_file = runtests::JoinPath(result->name, output_file_basename);
size_t i = strspn(output_file.c_str(), "/");
if (i == output_file.size()) {
fprintf(stderr, "Error: output_file was empty or all slashes: %s\n", output_file.c_str());
return EINVAL;
}
fprintf(summary_json, " \"output_file\": \"%s\",\n", &(output_file.c_str()[i]));
// Write the result of the test, which is either PASS or FAIL. We only
// have one PASS condition in TestResult, which is SUCCESS.
fprintf(summary_json, " \"result\": \"%s\",\n",
result->launch_status == runtests::SUCCESS ? "PASS" : "FAIL");
fprintf(summary_json, " \"duration_milliseconds\": %" PRId64,
result->duration_milliseconds);
// Write all data sinks.
if (result->data_sinks.size()) {
fprintf(summary_json, ",\n \"data_sinks\": {\n");
int sink_count = 0;
for (const auto& sink : result->data_sinks) {
if (sink_count != 0) {
fprintf(summary_json, ",\n");
}
fprintf(summary_json, " \"%s\": [\n", sink.first.c_str());
int file_count = 0;
for (const auto& file : sink.second) {
if (file_count != 0) {
fprintf(summary_json, ",\n");
}
fprintf(summary_json,
" {\n"
" \"name\": \"%s\",\n"
" \"file\": \"%s\"\n"
" }",
file.name.c_str(), file.file.c_str());
file_count++;
}
fprintf(summary_json, "\n ]");
sink_count++;
}
fprintf(summary_json, "\n }\n");
} else {
fprintf(summary_json, "\n");
}
fprintf(summary_json, " }");
test_count++;
}
fprintf(summary_json, "\n ]");
if (!syslog_path.empty()) {
fprintf(summary_json,
",\n"
" \"outputs\": {\n"
" \"syslog_file\": \"%.*s\"\n"
" }\n",
static_cast<int>(syslog_path.length()), syslog_path.data());
} else {
fprintf(summary_json, "\n");
}
// We really should have been checking for errors all along, but most likely the final fprintf
// will fail or succeed along with all the others.
const int final_ret_val = fprintf(summary_json, "}\n");
if (final_ret_val < 0) {
return errno;
}
return 0;
}
int ResolveGlobs(const fbl::Vector<fbl::String>& globs, fbl::Vector<fbl::String>* resolved) {
glob_t resolved_glob;
auto auto_call_glob_free = fbl::MakeAutoCall([&resolved_glob] { globfree(&resolved_glob); });
int flags = 0;
for (const auto& test_dir_glob : globs) {
int err = glob(test_dir_glob.c_str(), flags, nullptr, &resolved_glob);
// Ignore a lack of matches.
if (err && err != GLOB_NOMATCH) {
return err;
}
flags = GLOB_APPEND;
}
resolved->reserve(resolved_glob.gl_pathc);
for (size_t i = 0; i < resolved_glob.gl_pathc; ++i) {
resolved->push_back(fbl::String(resolved_glob.gl_pathv[i]));
}
return 0;
}
bool IsSharedLibraryName(fbl::StringPiece filename) {
struct ExcludePattern {
fbl::StringPiece prefix;
fbl::StringPiece suffix;
};
static const fbl::Vector<ExcludePattern> kExcludePatterns = {
{"lib", ".so"},
{"lib", ".dylib"},
};
for (const auto& exclusion : kExcludePatterns) {
if (filename.length() < exclusion.prefix.length() + exclusion.suffix.length()) {
continue;
}
fbl::StringPiece start(filename.begin(), exclusion.prefix.length());
fbl::StringPiece finish(filename.end() - exclusion.suffix.length(), exclusion.suffix.length());
if (start == exclusion.prefix && finish == exclusion.suffix) {
return true;
}
}
return false;
}
int DiscoverTestsInDirGlobs(const fbl::Vector<fbl::String>& dir_globs, const char* ignore_dir_name,
const fbl::Vector<fbl::String>& basename_whitelist,
fbl::Vector<fbl::String>* test_paths) {
fbl::Vector<fbl::String> test_dirs;
const int err = ResolveGlobs(dir_globs, &test_dirs);
if (err) {
fprintf(stderr, "Error: Failed to resolve globs, error = %d\n", err);
return EIO; // glob()'s return values aren't the same as errno. This is somewhat arbitrary.
}
for (const fbl::String& test_dir : test_dirs) {
// In the event of failures around a directory not existing or being an empty node
// we will continue to the next entries rather than aborting. This allows us to handle
// different sets of default test directories.
struct stat st;
if (stat(test_dir.c_str(), &st) < 0) {
printf("Could not stat %s, skipping...\n", test_dir.c_str());
continue;
}
if (!S_ISDIR(st.st_mode)) {
// Silently skip non-directories, as they may have been picked up in
// the glob.
continue;
}
// Resolve an absolute path to the test directory to ensure output
// directory names will never collide.
char abs_test_dir[PATH_MAX];
if (realpath(test_dir.c_str(), abs_test_dir) == nullptr) {
printf("Error: Could not resolve path %s: %s\n", test_dir.c_str(), strerror(errno));
continue;
}
// Silently skip |ignore_dir_name|.
// The user may have done something like runtests /foo/bar/h*.
const auto test_dir_base = basename(abs_test_dir);
if (ignore_dir_name && strcmp(test_dir_base, ignore_dir_name) == 0) {
continue;
}
DIR* dir = opendir(abs_test_dir);
if (dir == nullptr) {
fprintf(stderr, "Error: Could not open test dir %s\n", abs_test_dir);
return errno;
}
struct dirent* de;
while ((de = readdir(dir)) != nullptr) {
fbl::StringPiece test_name = de->d_name;
if (!basename_whitelist.is_empty() &&
!runtests::IsInWhitelist(test_name, basename_whitelist)) {
continue;
}
if (IsSharedLibraryName(test_name)) {
continue;
}
const fbl::String test_path = runtests::JoinPath(abs_test_dir, test_name);
if (stat(test_path.c_str(), &st) != 0 || !S_ISREG(st.st_mode)) {
continue;
}
test_paths->push_back(test_path);
}
closedir(dir);
}
return 0;
}
bool RunTests(const fbl::Vector<fbl::String>& test_paths, const fbl::Vector<fbl::String>& test_args,
int repeat, uint64_t timeout_msec, const char* output_dir,
const fbl::StringPiece output_file_basename, const char* realm_label,
int* failed_count, fbl::Vector<std::unique_ptr<Result>>* results) {
std::map<fbl::String, int> test_name_to_count;
for (int i = 1; i <= repeat; ++i) {
for (const fbl::String& test_path : test_paths) {
fbl::String output_filename_str;
fbl::String output_test_name;
if (test_name_to_count.find(test_path) != test_name_to_count.end()) {
int count = test_name_to_count[test_path];
count += 1;
output_test_name = fbl::String::Concat({test_path, " (", std::to_string(count), ")"});
test_name_to_count[test_path] = count;
} else {
test_name_to_count[test_path] = 1;
output_test_name = test_path;
}
// Ensure the output directory for this test binary's output exists.
if (output_dir != nullptr) {
// If output_dir was specified, ask |RunTest| to redirect stdout/stderr
// to a file whose name is based on the test name.
fbl::String output_dir_for_test_str = runtests::JoinPath(output_dir, output_test_name);
const int error = runtests::MkDirAll(output_dir_for_test_str);
if (error) {
fprintf(stderr, "Error: Could not create output directory %s: %s\n",
output_dir_for_test_str.c_str(), strerror(error));
return false;
}
output_filename_str = JoinPath(output_dir_for_test_str, output_file_basename);
}
// Assemble test binary args.
fbl::Vector<const char*> argv;
argv.push_back(test_path.c_str());
argv.reserve(test_args.size());
for (auto test_arg = test_args.begin(); test_arg != test_args.end(); ++test_arg) {
argv.push_back(test_arg->c_str());
}
argv.push_back(nullptr); // Important, since there's no argc.
const char* output_filename = nullptr;
if (!output_filename_str.empty()) {
output_filename = output_filename_str.c_str();
// Ensure the output file exists, even if RunTest fails to create it.
// This makes it possible for the infra to trust the summary.json.
FILE* output_file = fopen(output_filename, "w");
if (output_file == nullptr) {
fprintf(stderr, "Error: Could not create output file %s: %s\n", output_filename,
strerror(errno));
return false;
}
fclose(output_file);
}
// Execute the test binary.
printf(
"\n------------------------------------------------\n"
"RUNNING TEST: %s\n\n",
output_test_name.c_str());
fflush(stdout);
std::unique_ptr<Result> result = RunTest(argv.data(), output_dir, output_filename,
output_test_name.c_str(), timeout_msec, realm_label);
char duration_str[64]; // Size should be large enough for all reasonable durations.
snprintf(duration_str, sizeof(duration_str), "%" PRIu64 ".%03u sec",
result->duration_milliseconds / 1000,
(unsigned)(result->duration_milliseconds % 1000));
// testrunner looks for "<kFailureSignature|kSuccessSignature> <test name>". Anything after
// that is just for humans.
if (result->launch_status == SUCCESS) {
fprintf(stderr, "%s %s (%s)\n", kSuccessSignature, output_test_name.c_str(), duration_str);
} else {
*failed_count += 1;
fprintf(stderr, "%s %s (%s)\n", kFailureSignature, output_test_name.c_str(), duration_str);
}
results->push_back(std::move(result));
}
if (i < repeat) {
printf("\nPROGRESS: Ran %lu tests: %d failed\n", results->size(), *failed_count);
}
}
return true;
}
bool IsFuchsiaPkgURI(const char* s) {
const char prefix[] = "fuchsia-pkg://";
const size_t prefix_len = sizeof(prefix) - 1;
return 0 == strncmp(s, prefix, prefix_len);
}
} // namespace runtests