Add spec runner C++ script
177.mesa and 164.gzip are currently using this framework to run the
benchmarks
Also adding clang format config file
Change-Id: I85c90cb59d05d11f5334067de6614770bee30f6b
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000..b17a52a
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,2 @@
+BasedOnStyle: Chromium
+Standard: Cpp11
diff --git a/BUILD.gn b/BUILD.gn
index a0c45f6..aa1418e 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -6,6 +6,7 @@
testonly = true
deps = [
+ "benchmarks:speccpu2000_benchmark",
"164.gzip",
"175.vpr",
"176.gcc",
diff --git a/benchmarks/BUILD.gn b/benchmarks/BUILD.gn
new file mode 100644
index 0000000..e6ec77e
--- /dev/null
+++ b/benchmarks/BUILD.gn
@@ -0,0 +1,42 @@
+# Copyright 2016 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+config("util_config") {
+ visibility = [ ":*" ]
+ include_dirs = [ "." ]
+}
+
+executable("speccpu2000_benchmark") {
+ testonly = true
+
+ sources = [
+ "main.cc",
+ ]
+
+ deps = [
+ ":spec_utils",
+ "//third_party/benchmark",
+ "//third_party/glog",
+ ]
+}
+
+source_set("spec_utils") {
+ testonly = true
+
+ sources = [
+ "spec_fixture.cc",
+ "spec_utils.cc",
+ ]
+
+ deps = [
+ "//third_party/benchmark",
+ "//third_party/glog",
+ ]
+
+ libs = [
+ "launchpad",
+ "magenta",
+ ]
+ public_configs = [ ":util_config" ]
+}
diff --git a/benchmarks/main.cc b/benchmarks/main.cc
new file mode 100644
index 0000000..38712c1
--- /dev/null
+++ b/benchmarks/main.cc
@@ -0,0 +1,81 @@
+// Copyright 2016 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <benchmark/benchmark.h>
+#include <fstream>
+#include <libgen.h>
+#include <spec_fixture.h>
+#include <spec_utils.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <glog/logging.h>
+
+namespace {
+
+class MesaFixture : public SpecFixture {
+ protected:
+ MesaFixture() : SpecFixture("177.mesa") {}
+};
+
+class GzipFixture : public SpecFixture {
+ protected:
+ GzipFixture() : SpecFixture("164.gzip"){};
+};
+
+} // namespace
+
+std::string executableDir;
+
+// Where we will run benchmarks
+std::string tmpDir;
+
+BENCHMARK_F(MesaFixture, 177_Mesa)(benchmark::State& st) {
+ while (st.KeepRunning()) {
+ if (RunSpec(NULL, 0) != 0) {
+ st.SkipWithError(
+ (std::string("Error while running benchmark: ") + strerror(errno))
+ .c_str());
+ }
+ }
+}
+
+BENCHMARK_F(GzipFixture, 164_Gzip)(benchmark::State& st) {
+ while (st.KeepRunning()) {
+ std::vector<std::string> inputs = {"input.source", "input.log",
+ "input.graphic", "input.random",
+ "input.program"};
+
+ for (auto& it : inputs) {
+ const char* args[] = {it.c_str(), "60"};
+ if (RunSpec(args, 2) != 0) {
+ st.SkipWithError(
+ (std::string("Error while running benchmark: ") + strerror(errno))
+ .c_str());
+ }
+ }
+ }
+}
+
+int main(int argc, char* argv[]) {
+ ::benchmark::Initialize(&argc, argv);
+ google::InitGoogleLogging(argv[0]);
+ char* cwd;
+ CHECK_NOTNULL((cwd = get_current_dir_name()));
+ executableDir = std::string(cwd) + "/" + dirname(argv[0]);
+
+ char* c = tempnam(NULL, "speccpu2000");
+ CHECK_NOTNULL(c);
+ tmpDir = c;
+
+ CHECK_EQ(mkdir(tmpDir.c_str(), 0600), 0)
+ << "Error while creating tmp dir: " << tmpDir.c_str()
+ << ", error:" << strerror(errno);
+
+ ::benchmark::RunSpecifiedBenchmarks();
+
+ // try deleting temp directory
+ CHECK_EQ(DeleteDir(tmpDir), 0) << "Error while deleting tmp dir: " << tmpDir
+ << ", error:" << strerror(errno);
+ return 0;
+}
diff --git a/benchmarks/spec_fixture.cc b/benchmarks/spec_fixture.cc
new file mode 100644
index 0000000..befc417
--- /dev/null
+++ b/benchmarks/spec_fixture.cc
@@ -0,0 +1,147 @@
+// Copyright 2016 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "spec_fixture.h"
+#include "spec_utils.h"
+
+#include <benchmark/benchmark.h>
+#include <fcntl.h>
+#include <fstream>
+#include <glog/logging.h>
+#include <launchpad/launchpad.h>
+#include <magenta/syscalls.h>
+#include <magenta/syscalls/object.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+extern std::string executableDir;
+extern std::string tmpDir;
+extern std::ofstream result;
+
+namespace {
+class ScopedGuard {
+ public:
+ explicit ScopedGuard(std::function<void()> f) : f_(f) {}
+ ~ScopedGuard() noexcept { f_(); }
+ ScopedGuard(const ScopedGuard&) = delete;
+ ScopedGuard(ScopedGuard&&) = delete;
+ ScopedGuard& operator=(const ScopedGuard&) = delete;
+ ScopedGuard& operator=(ScopedGuard&&) = delete;
+
+ private:
+ std::function<void()> f_;
+};
+} // namespace
+
+void SpecFixture::SetUp(benchmark::State& st) {
+ DLOG(INFO) << "Preparing benchmark: " << benchmark_name;
+ run_dir_name = tmpDir + "/" + benchmark_name;
+
+ DLOG(INFO) << "Changing dir to: " << executableDir.c_str();
+ CHECK_EQ(chdir(executableDir.c_str()), 0) << "Not able to change dir to "
+ << executableDir;
+
+ CHECK_EQ(mkdir(run_dir_name.c_str(), 0600), 0)
+ << "Not able to create dir: " << run_dir_name
+ << ", Error: " << strerror(errno);
+
+ CHECK_EQ(CopyDir(run_dir_name, "data/" + benchmark_name), 0)
+ << "Error copying dir: " << run_dir_name
+ << ", Error: " << strerror(errno);
+
+ CHECK_EQ(CopyFile(run_dir_name + "/" + benchmark_name, benchmark_name), 0)
+ << "Error copying binary, Error: " << strerror(errno);
+}
+
+void SpecFixture::TearDown(benchmark::State& st) {
+ CHECK_EQ(DeleteDir(run_dir_name), 0) << "Error Deleting directory "
+ << run_dir_name
+ << ", Error: " << strerror(errno);
+}
+
+int SpecFixture::RunSpec(const char* args[], int args_length) {
+ char** argv = new char*[args_length + 1];
+ argv[0] = (char*)(benchmark_name.c_str());
+ if (args_length > 0) {
+ memcpy(argv + 1, args, args_length * sizeof(char*));
+ }
+
+ if (chdir(run_dir_name.c_str()) != 0) {
+ LOG(ERROR) << "Not able to change dir to " << run_dir_name
+ << ", Error: " << strerror(errno);
+ return errno;
+ }
+
+ DLOG(INFO) << "Running benchmark";
+
+ // Capture output
+ char buffer[PAGE_SIZE];
+ int out_p[2];
+ int old_stdout = dup(STDOUT_FILENO);
+ if (pipe2(out_p, O_NONBLOCK) != 0) {
+ LOG(ERROR) << "Not able create pipe, Error: " << strerror(errno);
+ return errno;
+ }
+ ScopedGuard guard([&]() { close(out_p[0]); });
+ dup2(out_p[1], STDOUT_FILENO);
+ close(out_p[1]);
+
+ int status = LaunchProcess(argv, args_length + 1);
+
+ fflush(stdout);
+ dup2(old_stdout, STDOUT_FILENO);
+ if (chdir(executableDir.c_str()) != 0) {
+ LOG(ERROR) << "Not able to change dir to " << executableDir
+ << ", Error: " << strerror(errno);
+ return errno;
+ }
+ delete[] argv;
+ if (status != 0) {
+ // Dump output
+ LOG(ERROR) << "Error while running benchmark";
+ ssize_t len;
+ do {
+ len = read(out_p[0], buffer, PAGE_SIZE);
+ if (len == -1) {
+ LOG(ERROR) << "Error reading from pipe, Error: " << strerror(errno);
+ return errno;
+ }
+ fwrite(buffer, sizeof(char), len, stderr);
+ } while (len == PAGE_SIZE);
+ fprintf(stderr, "\n");
+ return 1;
+ }
+ return 0;
+}
+
+// TODO: This is for fuchsia, need to write linux specific
+#ifdef __Fuchsia__
+int SpecFixture::LaunchProcess(char** args, int args_length) {
+ mx_handle_t handle = launchpad_launch_mxio(args[0], args_length, args);
+ if (handle < 0) {
+ LOG(ERROR) << "Failed to launch " << args[0] << ":" << handle;
+ return handle;
+ }
+ mx_status_t status =
+ mx_handle_wait_one(handle, MX_SIGNAL_SIGNALED, MX_TIME_INFINITE, NULL);
+
+ if (status != NO_ERROR) {
+ LOG(ERROR) << "Failed to wait for process exiting " << args[0] << ":"
+ << status;
+ return status;
+ }
+
+ mx_info_process_t proc_info;
+ ssize_t info_status = mx_object_get_info(handle, MX_INFO_PROCESS, &proc_info,
+ sizeof(proc_info), NULL, NULL);
+ mx_handle_close(handle);
+
+ if (info_status < 0) {
+ LOG(ERROR) << "Failed to get process return code " << args[0] << ":"
+ << info_status;
+ return 1;
+ }
+ return proc_info.return_code;
+}
+#endif
diff --git a/benchmarks/spec_fixture.h b/benchmarks/spec_fixture.h
new file mode 100644
index 0000000..cdeacde
--- /dev/null
+++ b/benchmarks/spec_fixture.h
@@ -0,0 +1,28 @@
+// Copyright 2016 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#pragma once
+
+#include <benchmark/benchmark.h>
+#include <string.h>
+
+class SpecFixture : public benchmark::Fixture {
+ private:
+ std::string run_dir_name;
+
+ // Launch process defined by arg. It returns 1 if execution failure else
+ // returns 0.
+ int LaunchProcess(char** args, int args_length);
+
+ protected:
+ std::string benchmark_name;
+
+ SpecFixture(std::string name) : benchmark_name(name){};
+
+ void SetUp(benchmark::State& st) override;
+
+ void TearDown(benchmark::State& st) override;
+
+ int RunSpec(const char* args[], int args_length);
+};
diff --git a/benchmarks/spec_utils.cc b/benchmarks/spec_utils.cc
new file mode 100644
index 0000000..945761a
--- /dev/null
+++ b/benchmarks/spec_utils.cc
@@ -0,0 +1,122 @@
+// Copyright 2016 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "spec_utils.h"
+#include <dirent.h>
+#include <errno.h>
+#include <glog/logging.h>
+#include <launchpad/launchpad.h>
+#include <limits.h>
+#include <magenta/syscalls.h>
+#include <string>
+#include <sys/stat.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+int DeleteDir(std::string dirname) {
+ DLOG(INFO) << "Deleting: " << dirname;
+ DIR* dir = opendir(dirname.c_str());
+ if (dir == NULL) {
+ return errno;
+ }
+
+ struct dirent* de;
+ struct stat stat_buf;
+ while ((de = readdir(dir)) != NULL) {
+ if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) {
+ continue;
+ }
+
+ std::string name = dirname + "/" + de->d_name;
+ if (stat(name.c_str(), &stat_buf) == 0 && S_ISDIR(stat_buf.st_mode)) {
+ if (DeleteDir(name) != 0) {
+ return errno;
+ }
+ } else {
+ if (unlink(name.c_str()) != 0) {
+ return errno;
+ }
+ }
+ }
+
+ if (closedir(dir) != 0) {
+ return errno;
+ }
+ if (rmdir(dirname.c_str()) != 0) {
+ return errno;
+ }
+ return 0;
+}
+
+int CopyDir(std::string dest_dirname, std::string src_dirname) {
+ DIR* dir = opendir(src_dirname.c_str());
+ if (dir == NULL) {
+ return errno;
+ }
+
+ DLOG(INFO) << "Copy " << src_dirname << " to " << dest_dirname;
+ struct dirent* de;
+ struct stat stat_buf;
+ while ((de = readdir(dir)) != NULL) {
+ std::string name = src_dirname + "/" + de->d_name;
+ if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) {
+ continue;
+ }
+
+ if (stat(name.c_str(), &stat_buf) != 0) {
+ return errno;
+ }
+ if (S_ISDIR(stat_buf.st_mode)) {
+ std::string sub_dir = dest_dirname + "/" + de->d_name;
+ if (mkdir(sub_dir.c_str(), 0600) != 0) {
+ return errno;
+ }
+ if (CopyDir(sub_dir, name) != 0) {
+ return errno;
+ }
+ } else if (S_ISREG(stat_buf.st_mode)) {
+ if (CopyFile(dest_dirname + "/" + de->d_name, name) != 0) {
+ return errno;
+ }
+ }
+ }
+ if (closedir(dir) != 0) {
+ return errno;
+ }
+ return 0;
+}
+
+int CopyFile(std::string dest, std::string src) {
+ char buf[PAGE_SIZE];
+ size_t size;
+ DLOG(INFO) << "Copy " << src << " to " << dest;
+ FILE* srcfile = fopen(src.c_str(), "rb");
+ if (srcfile == NULL) {
+ return errno;
+ }
+ FILE* destfile = fopen(dest.c_str(), "wb");
+ if (destfile == NULL) {
+ return errno;
+ }
+
+ while ((size = fread(buf, 1, PAGE_SIZE, srcfile))) {
+ if (fwrite(buf, 1, size, destfile) != size) {
+ return errno;
+ }
+ }
+
+ if (fclose(srcfile) != 0) {
+ return errno;
+ }
+ if (fflush(destfile) != 0) {
+ return errno;
+ }
+ if (fclose(destfile) != 0) {
+ return errno;
+ }
+ return 0;
+}
diff --git a/benchmarks/spec_utils.h b/benchmarks/spec_utils.h
new file mode 100644
index 0000000..0141607
--- /dev/null
+++ b/benchmarks/spec_utils.h
@@ -0,0 +1,18 @@
+// Copyright 2016 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#pragma once
+
+#include <string>
+#include <sys/time.h>
+
+// Deletes Directory recursively.
+// Retunrs errno if error in deleting
+int DeleteDir(std::string dirname);
+
+// Copies directory recursively.
+int CopyDir(std::string dest_dirname, std::string src_dirname);
+
+// Copies content of the file.
+int CopyFile(std::string dest, std::string src);
