Add a per benchmark 'Repetitions' option.
diff --git a/README.md b/README.md
index 498c4ca..b03d222 100644
--- a/README.md
+++ b/README.md
@@ -279,6 +279,17 @@
set as a flag `--benchmark_min_time` or per benchmark by calling `MinTime` on
the registered benchmark object.
+## Reporting the mean and standard devation by repeated benchmarks
+By default each benchmark is run once and that single result is reported.
+However benchmarks are often noisy and a single result may not be representative
+of the overall behavior. For this reason it's possible to repeatedly rerun the
+benchmark.
+
+The number of runs of each benchmark is specified globally by the
+`--benchmark_repetitions` flag or on a per benchmark basis by calling
+`Repetitions` on the registered benchmark object. When a benchmark is run
+more than once the mean and standard deviation of the runs will be reported.
+
## Fixtures
Fixture tests are created by
first defining a type that derives from ::benchmark::Fixture and then
diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h
index 674b0b5..2fe8753 100644
--- a/include/benchmark/benchmark_api.h
+++ b/include/benchmark/benchmark_api.h
@@ -477,8 +477,14 @@
// Set the minimum amount of time to use when running this benchmark. This
// option overrides the `benchmark_min_time` flag.
+ // REQUIRES: `t > 0`
Benchmark* MinTime(double t);
+ // Specify the amount of times to repeat this benchmark. This option overrides
+ // the `benchmark_repetitions` flag.
+ // REQUIRES: `n > 0`
+ Benchmark* Repetitions(int n);
+
// If a particular benchmark is I/O bound, runs multiple threads internally or
// if for some reason CPU timings are not representative, call this method. If
// called, the elapsed time will be used to control how many iterations are
diff --git a/src/benchmark.cc b/src/benchmark.cc
index bd9858f..41d529d 100644
--- a/src/benchmark.cc
+++ b/src/benchmark.cc
@@ -293,6 +293,7 @@
bool use_manual_time;
BigO complexity;
bool last_benchmark_instance;
+ int repetitions;
double min_time;
int threads; // Number of concurrent threads to use
bool multithreaded; // Is benchmark multi-threaded?
@@ -332,6 +333,7 @@
void RangePair(int lo1, int hi1, int lo2, int hi2);
void RangeMultiplier(int multiplier);
void MinTime(double n);
+ void Repetitions(int n);
void UseRealTime();
void UseManualTime();
void Complexity(BigO complexity);
@@ -351,6 +353,7 @@
TimeUnit time_unit_;
int range_multiplier_;
double min_time_;
+ int repetitions_;
bool use_real_time_;
bool use_manual_time_;
BigO complexity_;
@@ -414,6 +417,7 @@
instance.time_unit = family->time_unit_;
instance.range_multiplier = family->range_multiplier_;
instance.min_time = family->min_time_;
+ instance.repetitions = family->repetitions_;
instance.use_real_time = family->use_real_time_;
instance.use_manual_time = family->use_manual_time_;
instance.complexity = family->complexity_;
@@ -430,6 +434,9 @@
if (!IsZero(family->min_time_)) {
instance.name += StringPrintF("/min_time:%0.3f", family->min_time_);
}
+ if (family->repetitions_ != 0) {
+ instance.name += StringPrintF("/repeats:%d", family->repetitions_);
+ }
if (family->use_manual_time_) {
instance.name += "/manual_time";
} else if (family->use_real_time_) {
@@ -453,7 +460,7 @@
BenchmarkImp::BenchmarkImp(const char* name)
: name_(name), arg_count_(-1), time_unit_(kNanosecond),
- range_multiplier_(kRangeMultiplier), min_time_(0.0),
+ range_multiplier_(kRangeMultiplier), min_time_(0.0), repetitions_(0),
use_real_time_(false), use_manual_time_(false),
complexity_(oNone) {
}
@@ -521,6 +528,12 @@
min_time_ = t;
}
+
+void BenchmarkImp::Repetitions(int n) {
+ CHECK(n > 0);
+ repetitions_ = n;
+}
+
void BenchmarkImp::UseRealTime() {
CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously.";
use_real_time_ = true;
@@ -633,6 +646,12 @@
return this;
}
+
+Benchmark* Benchmark::Repetitions(int t) {
+ imp_->Repetitions(t);
+ return this;
+}
+
Benchmark* Benchmark::MinTime(double t) {
imp_->MinTime(t);
return this;
@@ -712,7 +731,9 @@
if (b.multithreaded)
pool.resize(b.threads);
- for (int i = 0; i < FLAGS_benchmark_repetitions; i++) {
+ const int repeats = b.repetitions != 0 ? b.repetitions
+ : FLAGS_benchmark_repetitions;
+ for (int i = 0; i < repeats; i++) {
std::string mem;
for (;;) {
// Try benchmark
@@ -893,12 +914,14 @@
CHECK(reporter != nullptr);
// Determine the width of the name field using a minimum width of 10.
+ bool has_repetitions = FLAGS_benchmark_repetitions > 1;
size_t name_field_width = 10;
for (const Benchmark::Instance& benchmark : benchmarks) {
name_field_width =
std::max<size_t>(name_field_width, benchmark.name.size());
+ has_repetitions |= benchmark.repetitions > 1;
}
- if (FLAGS_benchmark_repetitions > 1)
+ if (has_repetitions)
name_field_width += std::strlen("_stddev");
// Print header here
diff --git a/test/options_test.cc b/test/options_test.cc
index ed5fa68..78cedae 100644
--- a/test/options_test.cc
+++ b/test/options_test.cc
@@ -31,6 +31,7 @@
BENCHMARK(BM_basic)->UseRealTime();
BENCHMARK(BM_basic)->ThreadRange(2, 4);
BENCHMARK(BM_basic)->ThreadPerCpu();
+BENCHMARK(BM_basic)->Repetitions(3);
void CustomArgs(benchmark::internal::Benchmark* b) {
for (int i = 0; i < 10; ++i) {
