| //! A statistics-driven micro-benchmarking library written in Rust. |
| //! |
| //! This crate is a microbenchmarking library which aims to provide strong |
| //! statistical confidence in detecting and estimating the size of performance |
| //! improvements and regressions, whle also being easy to use. |
| //! |
| //! See |
| //! [the user guide](https://bheisler.github.io/criterion.rs/book/index.html) |
| //! for examples as well as details on the measurement and analysis process, |
| //! and the output. |
| //! |
| //! ## Features: |
| //! * Benchmark Rust code as well as external programs |
| //! * Collects detailed statistics, providing strong confidence that changes |
| //! to performance are real, not measurement noise |
| //! * Produces detailed charts, providing thorough understanding of your code's |
| //! performance behavior. |
| |
| #![deny(missing_docs)] |
| #![cfg_attr(feature = "real_blackbox", feature(test))] |
| #![cfg_attr(not(feature = "html_reports"), allow(dead_code))] |
| #![cfg_attr( |
| feature = "cargo-clippy", |
| allow( |
| clippy::used_underscore_binding, |
| clippy::just_underscores_and_digits, |
| clippy::transmute_ptr_to_ptr |
| ) |
| )] |
| |
| #[cfg(test)] |
| #[macro_use] |
| extern crate approx; |
| |
| #[cfg(test)] |
| #[macro_use] |
| extern crate quickcheck; |
| |
| #[cfg(test)] |
| extern crate rand; |
| |
| #[macro_use] |
| extern crate clap; |
| |
| #[macro_use] |
| extern crate lazy_static; |
| |
| extern crate atty; |
| extern crate cast; |
| extern crate csv; |
| extern crate itertools; |
| extern crate num_traits; |
| extern crate rand_core; |
| extern crate rand_os; |
| extern crate rand_xoshiro; |
| extern crate rayon; |
| extern crate serde; |
| extern crate serde_json; |
| extern crate walkdir; |
| |
| #[cfg(feature = "html_reports")] |
| extern crate criterion_plot; |
| |
| #[cfg(feature = "html_reports")] |
| extern crate tinytemplate; |
| |
| #[cfg(feature = "real_blackbox")] |
| extern crate test; |
| |
| #[macro_use] |
| extern crate serde_derive; |
| |
| // Needs to be declared before other modules |
| // in order to be usable there. |
| #[macro_use] |
| mod macros_private; |
| #[macro_use] |
| mod analysis; |
| mod benchmark; |
| mod csv_report; |
| mod error; |
| mod estimate; |
| mod format; |
| mod fs; |
| mod macros; |
| mod program; |
| mod report; |
| mod routine; |
| mod stats; |
| |
| #[cfg(feature = "html_reports")] |
| mod kde; |
| |
| #[cfg(feature = "html_reports")] |
| mod plot; |
| |
| #[cfg(feature = "html_reports")] |
| mod html; |
| |
| use std::cell::RefCell; |
| use std::collections::BTreeMap; |
| use std::default::Default; |
| use std::fmt; |
| use std::iter::IntoIterator; |
| use std::process::Command; |
| use std::time::{Duration, Instant}; |
| |
| use benchmark::BenchmarkConfig; |
| use benchmark::NamedRoutine; |
| use csv_report::FileCsvReport; |
| use estimate::{Distributions, Estimates, Statistic}; |
| use plotting::Plotting; |
| use report::{CliReport, Report, ReportContext, Reports}; |
| use routine::Function; |
| |
| #[cfg(feature = "html_reports")] |
| use html::Html; |
| |
| pub use benchmark::{Benchmark, BenchmarkDefinition, ParameterizedBenchmark}; |
| |
| lazy_static! { |
| static ref DEBUG_ENABLED: bool = { std::env::vars().any(|(key, _)| key == "CRITERION_DEBUG") }; |
| } |
| |
| fn debug_enabled() -> bool { |
| *DEBUG_ENABLED |
| } |
| |
| // Fake function which shows a deprecation warning when compiled without the html_reports feature. |
| #[cfg(not(feature = "html_reports"))] |
| #[cfg_attr(not(feature = "html_reports"), doc(hidden))] |
| pub fn deprecation_warning() { |
| #[deprecated( |
| since = "0.2.6", |
| note = "The html_reports cargo feature is deprecated. As of 0.3.0, HTML reports will no longer be optional." |
| )] |
| fn deprecation_warning_inner() {} |
| |
| deprecation_warning_inner() |
| } |
| |
| /// A function that is opaque to the optimizer, used to prevent the compiler from |
| /// optimizing away computations in a benchmark. |
| /// |
| /// This variant is backed by the (unstable) test::black_box function. |
| #[cfg(feature = "real_blackbox")] |
| pub fn black_box<T>(dummy: T) -> T { |
| test::black_box(dummy) |
| } |
| |
| /// A function that is opaque to the optimizer, used to prevent the compiler from |
| /// optimizing away computations in a benchmark. |
| /// |
| /// This variant is stable-compatible, but it may cause some performance overhead |
| /// or fail to prevent code from being eliminated. |
| #[cfg(not(feature = "real_blackbox"))] |
| pub fn black_box<T>(dummy: T) -> T { |
| unsafe { |
| let ret = std::ptr::read_volatile(&dummy); |
| std::mem::forget(dummy); |
| ret |
| } |
| } |
| |
| /// Representing a function to benchmark together with a name of that function. |
| /// Used together with `bench_functions` to represent one out of multiple functions |
| /// under benchmark. |
| pub struct Fun<I: fmt::Debug> { |
| f: NamedRoutine<I>, |
| } |
| |
| impl<I> Fun<I> |
| where |
| I: fmt::Debug + 'static, |
| { |
| /// Create a new `Fun` given a name and a closure |
| pub fn new<F>(name: &str, f: F) -> Fun<I> |
| where |
| F: FnMut(&mut Bencher, &I) + 'static, |
| { |
| let routine = NamedRoutine { |
| id: name.to_owned(), |
| f: Box::new(RefCell::new(Function::new(f))), |
| }; |
| |
| Fun { f: routine } |
| } |
| } |
| |
| /// Argument to [`Bencher::iter_batched`](struct.Bencher.html#method.iter_batched) and |
| /// [`Bencher::iter_batched_ref`](struct.Bencher.html#method.iter_batched_ref) which controls the |
| /// batch size. |
| /// |
| /// Generally speaking, almost all benchmarks should use `SmallInput`. If the input or the result |
| /// of the benchmark routine is large enough that `SmallInput` causes out-of-memory errors, |
| /// `LargeInput` can be used to reduce memory usage at the cost of increasing the measurement |
| /// overhead. If the input or the result is extremely large (or if it holds some |
| /// limited external resource like a file handle), `PerIteration` will set the number of iterations |
| /// per batch to exactly one. `PerIteration` can increase the measurement overhead substantially |
| /// and should be avoided wherever possible. |
| /// |
| /// Each value lists an estimate of the measurement overhead. This is intended as a rough guide |
| /// to assist in choosing an option, it should not be relied upon. In particular, it is not valid |
| /// to subtract the listed overhead from the measurement and assume that the result represents the |
| /// true runtime of a function. The actual measurement overhead for your specific benchmark depends |
| /// on the details of the function you're benchmarking and the hardware and operating |
| /// system running the benchmark. |
| /// |
| /// With that said, if the runtime of your function is small relative to the measurement overhead |
| /// it will be difficult to take accurate measurements. In this situation, the best option is to use |
| /// [`Bencher::iter`](struct.Bencher.html#method.iter_batched_ref) which has next-to-zero measurement |
| /// overhead. |
| #[derive(Debug, Eq, PartialEq, Copy, Hash, Clone)] |
| pub enum BatchSize { |
| /// `SmallInput` indicates that the input to the benchmark routine (the value returned from |
| /// the setup routine) is small enough that millions of values can be safely held in memory. |
| /// Always prefer `SmallInput` unless the benchmark is using too much memory. |
| /// |
| /// In testing, the maximum measurement overhead from benchmarking with `SmallInput` is on the |
| /// order of 500 picoseconds. This is presented as a rough guide; your results may vary. |
| SmallInput, |
| |
| /// `LargeInput` indicates that the input to the benchmark routine or the value returned from |
| /// that routine is large. This will reduce the memory usage but increase the measurement |
| /// overhead. |
| /// |
| /// In testing, the maximum measurement overhead from benchmarking with `LargeInput` is on the |
| /// order of 750 picoseconds. This is presented as a rough guide; your results may vary. |
| LargeInput, |
| |
| /// `PerIteration` indicates that the input to the benchmark routine or the value returned from |
| /// that routine is extremely large or holds some limited resource, such that holding many values |
| /// in memory at once is infeasible. This provides the worst measurement overhead, but the |
| /// lowest memory usage. |
| /// |
| /// In testing, the maximum measurement overhead from benchmarking with `PerIteration` is on the |
| /// order of 350 nanoseconds or 350,000 picoseconds. This is presented as a rough guide; your |
| /// results may vary. |
| PerIteration, |
| |
| /// `NumBatches` will attempt to divide the iterations up into a given number of batches. |
| /// A larger number of batches (and thus smaller batches) will reduce memory usage but increase |
| /// measurement overhead. This allows the user to choose their own tradeoff between memory usage |
| /// and measurement overhead, but care must be taken in tuning the number of batches. Most |
| /// benchmarks should use `SmallInput` or `LargeInput` instead. |
| NumBatches(u64), |
| |
| /// `NumIterations` fixes the batch size to a constant number, specified by the user. This |
| /// allows the user to choose their own tradeoff between overhead and memory usage, but care must |
| /// be taken in tuning the batch size. In general, the measurement overhead of NumIterations |
| /// will be larger than that of `NumBatches`. Most benchmarks should use `SmallInput` or |
| /// `LargeInput` instead. |
| NumIterations(u64), |
| |
| #[doc(hidden)] |
| __NonExhaustive, |
| } |
| impl BatchSize { |
| /// Convert to a number of iterations per batch. |
| /// |
| /// We try to do a constant number of batches regardless of the number of iterations in this |
| /// sample. If the measurement overhead is roughly constant regardless of the number of |
| /// iterations the analysis of the results later will have an easier time separating the |
| /// measurement overhead from the benchmark time. |
| fn iters_per_batch(self, iters: u64) -> u64 { |
| match self { |
| BatchSize::SmallInput => (iters + 10 - 1) / 10, |
| BatchSize::LargeInput => (iters + 1000 - 1) / 1000, |
| BatchSize::PerIteration => 1, |
| BatchSize::NumBatches(batches) => (iters + batches - 1) / batches, |
| BatchSize::NumIterations(size) => size, |
| BatchSize::__NonExhaustive => panic!("__NonExhaustive is not a valid BatchSize."), |
| } |
| } |
| } |
| |
| /// Timer struct to iterate a benchmarked function and measure the runtime. |
| /// |
| /// This struct provides different timing loops as methods. Each timing loop provides a different |
| /// way to time a routine and each has advantages and disadvantages. |
| /// |
| /// * If your routine requires no per-iteration setup and returns a value with an expensive `drop` |
| /// method, use `iter_with_large_drop`. |
| /// * If your routine requires some per-iteration setup that shouldn't be timed, use `iter_batched` |
| /// or `iter_batched_ref`. See [`BatchSize`](enum.BatchSize.html) for a discussion of batch sizes. |
| /// If the setup value implements `Drop` and you don't want to include the `drop` time in the |
| /// measurement, use `iter_batched_ref`, otherwise use `iter_batched`. These methods are also |
| /// suitable for benchmarking routines which return a value with an expensive `drop` method, |
| /// but are more complex than `iter_with_large_drop`. |
| /// * Otherwise, use `iter`. |
| #[derive(Clone, Copy)] |
| pub struct Bencher { |
| iterated: bool, |
| iters: u64, |
| elapsed: Duration, |
| } |
| impl Bencher { |
| /// Times a `routine` by executing it many times and timing the total elapsed time. |
| /// |
| /// Prefer this timing loop when `routine` returns a value that doesn't have a destructor. |
| /// |
| /// # Timing model |
| /// |
| /// Note that the `Bencher` also times the time required to destroy the output of `routine()`. |
| /// Therefore prefer this timing loop when the runtime of `mem::drop(O)` is negligible compared |
| /// to the runtime of the `routine`. |
| /// |
| /// ```text |
| /// elapsed = Instant::now + iters * (routine + mem::drop(O) + Range::next) |
| /// ``` |
| /// |
| /// # Example |
| /// |
| /// ```rust |
| /// #[macro_use] extern crate criterion; |
| /// |
| /// use criterion::*; |
| /// |
| /// // The function to benchmark |
| /// fn foo() { |
| /// // ... |
| /// } |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// c.bench_function("iter", move |b| { |
| /// b.iter(|| foo()) |
| /// }); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| /// |
| #[inline(never)] |
| pub fn iter<O, R>(&mut self, mut routine: R) |
| where |
| R: FnMut() -> O, |
| { |
| self.iterated = true; |
| let start = Instant::now(); |
| for _ in 0..self.iters { |
| black_box(routine()); |
| } |
| self.elapsed = start.elapsed(); |
| } |
| |
| #[doc(hidden)] |
| pub fn iter_with_setup<I, O, S, R>(&mut self, setup: S, routine: R) |
| where |
| S: FnMut() -> I, |
| R: FnMut(I) -> O, |
| { |
| self.iter_batched(setup, routine, BatchSize::PerIteration); |
| } |
| |
| /// Times a `routine` by collecting its output on each iteration. This avoids timing the |
| /// destructor of the value returned by `routine`. |
| /// |
| /// WARNING: This requires `O(iters * mem::size_of::<O>())` of memory, and `iters` is not under the |
| /// control of the caller. If this causes out-of-memory errors, use `iter_batched` instead. |
| /// |
| /// # Timing model |
| /// |
| /// ``` text |
| /// elapsed = Instant::now + iters * (routine) + Iterator::collect::<Vec<_>> |
| /// ``` |
| /// |
| /// # Example |
| /// |
| /// ```rust |
| /// #[macro_use] extern crate criterion; |
| /// |
| /// use criterion::*; |
| /// |
| /// fn create_vector() -> Vec<u64> { |
| /// # vec![] |
| /// // ... |
| /// } |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// c.bench_function("with_drop", move |b| { |
| /// // This will avoid timing the Vec::drop. |
| /// b.iter_with_large_drop(|| create_vector()) |
| /// }); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| /// |
| #[doc(hidden)] |
| pub fn iter_with_large_drop<O, R>(&mut self, mut routine: R) |
| where |
| R: FnMut() -> O, |
| { |
| self.iter_batched(|| (), |_| routine(), BatchSize::SmallInput); |
| } |
| |
| #[doc(hidden)] |
| pub fn iter_with_large_setup<I, O, S, R>(&mut self, setup: S, routine: R) |
| where |
| S: FnMut() -> I, |
| R: FnMut(I) -> O, |
| { |
| self.iter_batched(setup, routine, BatchSize::NumBatches(1)); |
| } |
| |
| /// Times a `routine` that requires some input by generating a batch of input, then timing the |
| /// iteration of the benchmark over the input. See [`BatchSize`](struct.BatchSize.html) for |
| /// details on choosing the batch size. Use this when the routine must consume its input. |
| /// |
| /// For example, use this loop to benchmark sorting algorithms, because they require unsorted |
| /// data on each iteration. |
| /// |
| /// # Timing model |
| /// |
| /// ```text |
| /// elapsed = (Instant::now * num_batches) + (iters * (routine + O::drop)) + Vec::extend |
| /// ``` |
| /// |
| /// # Example |
| /// |
| /// ```rust |
| /// #[macro_use] extern crate criterion; |
| /// |
| /// use criterion::*; |
| /// |
| /// fn create_scrambled_data() -> Vec<u64> { |
| /// # vec![] |
| /// // ... |
| /// } |
| /// |
| /// // The sorting algorithm to test |
| /// fn sort(data: &mut [u64]) { |
| /// // ... |
| /// } |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// let data = create_scrambled_data(); |
| /// |
| /// c.bench_function("with_setup", move |b| { |
| /// // This will avoid timing the to_vec call. |
| /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput) |
| /// }); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| /// |
| #[inline(never)] |
| pub fn iter_batched<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize) |
| where |
| S: FnMut() -> I, |
| R: FnMut(I) -> O, |
| { |
| self.iterated = true; |
| let batch_size = size.iters_per_batch(self.iters); |
| assert!(batch_size != 0, "Batch size must not be zero."); |
| self.elapsed = Duration::from_secs(0); |
| |
| if batch_size == 1 { |
| for _ in 0..self.iters { |
| let mut input = black_box(setup()); |
| |
| let start = Instant::now(); |
| let output = routine(input); |
| self.elapsed += start.elapsed(); |
| |
| drop(black_box(output)); |
| } |
| } else { |
| let mut iteration_counter = 0; |
| |
| while iteration_counter < self.iters { |
| let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter); |
| |
| let inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>()); |
| let mut outputs = Vec::with_capacity(batch_size as usize); |
| |
| let start = Instant::now(); |
| outputs.extend(inputs.into_iter().map(&mut routine)); |
| self.elapsed += start.elapsed(); |
| |
| black_box(outputs); |
| |
| iteration_counter += batch_size; |
| } |
| } |
| } |
| |
| /// Times a `routine` that requires some input by generating a batch of input, then timing the |
| /// iteration of the benchmark over the input. See [`BatchSize`](struct.BatchSize.html) for |
| /// details on choosing the batch size. Use this when the routine should accept the input by |
| /// mutable reference. |
| /// |
| /// For example, use this loop to benchmark sorting algorithms, because they require unsorted |
| /// data on each iteration. |
| /// |
| /// # Timing model |
| /// |
| /// ```text |
| /// elapsed = (Instant::now * num_batches) + (iters * routine) + Vec::extend |
| /// ``` |
| /// |
| /// # Example |
| /// |
| /// ```rust |
| /// #[macro_use] extern crate criterion; |
| /// |
| /// use criterion::*; |
| /// |
| /// fn create_scrambled_data() -> Vec<u64> { |
| /// # vec![] |
| /// // ... |
| /// } |
| /// |
| /// // The sorting algorithm to test |
| /// fn sort(data: &mut [u64]) { |
| /// // ... |
| /// } |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// let data = create_scrambled_data(); |
| /// |
| /// c.bench_function("with_setup", move |b| { |
| /// // This will avoid timing the to_vec call. |
| /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput) |
| /// }); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| /// |
| #[inline(never)] |
| pub fn iter_batched_ref<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize) |
| where |
| S: FnMut() -> I, |
| R: FnMut(&mut I) -> O, |
| { |
| self.iterated = true; |
| let batch_size = size.iters_per_batch(self.iters); |
| assert!(batch_size != 0, "Batch size must not be zero."); |
| self.elapsed = Duration::from_secs(0); |
| |
| if batch_size == 1 { |
| for _ in 0..self.iters { |
| let mut input = black_box(setup()); |
| |
| let start = Instant::now(); |
| let output = routine(&mut input); |
| self.elapsed += start.elapsed(); |
| |
| drop(black_box(output)); |
| drop(black_box(input)); |
| } |
| } else { |
| let mut iteration_counter = 0; |
| |
| while iteration_counter < self.iters { |
| let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter); |
| |
| let mut inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>()); |
| let mut outputs = Vec::with_capacity(batch_size as usize); |
| |
| let start = Instant::now(); |
| outputs.extend(inputs.iter_mut().map(&mut routine)); |
| self.elapsed += start.elapsed(); |
| |
| black_box(outputs); |
| |
| iteration_counter += batch_size; |
| } |
| } |
| } |
| |
| // Benchmarks must actually call one of the iter methods. This causes benchmarks to fail loudly |
| // if they don't. |
| fn assert_iterated(&mut self) { |
| if !self.iterated { |
| panic!("Benchmark function must call Bencher::iter or related method."); |
| } |
| self.iterated = false; |
| } |
| } |
| |
| /// Baseline describes how the baseline_directory is handled. |
| pub enum Baseline { |
| /// Compare ensures a previous saved version of the baseline |
| /// exists and runs comparison against that. |
| Compare, |
| /// Save writes the benchmark results to the baseline directory, |
| /// overwriting any results that were previously there. |
| Save, |
| } |
| |
| /// The benchmark manager |
| /// |
| /// `Criterion` lets you configure and execute benchmarks |
| /// |
| /// Each benchmark consists of four phases: |
| /// |
| /// - **Warm-up**: The routine is repeatedly executed, to let the CPU/OS/JIT/interpreter adapt to |
| /// the new load |
| /// - **Measurement**: The routine is repeatedly executed, and timing information is collected into |
| /// a sample |
| /// - **Analysis**: The sample is analyzed and distiled into meaningful statistics that get |
| /// reported to stdout, stored in files, and plotted |
| /// - **Comparison**: The current sample is compared with the sample obtained in the previous |
| /// benchmark. |
| pub struct Criterion { |
| config: BenchmarkConfig, |
| plotting: Plotting, |
| filter: Option<String>, |
| report: Box<Report>, |
| output_directory: String, |
| baseline_directory: String, |
| baseline: Baseline, |
| profile_time: Option<Duration>, |
| test_mode: bool, |
| list_mode: bool, |
| } |
| |
| impl Default for Criterion { |
| /// Creates a benchmark manager with the following default settings: |
| /// |
| /// - Sample size: 100 measurements |
| /// - Warm-up time: 3 s |
| /// - Measurement time: 5 s |
| /// - Bootstrap size: 100 000 resamples |
| /// - Noise threshold: 0.01 (1%) |
| /// - Confidence level: 0.95 |
| /// - Significance level: 0.05 |
| /// - Plotting: enabled (if gnuplot is available) |
| /// - No filter |
| fn default() -> Criterion { |
| #[allow(unused_mut, unused_assignments)] |
| let mut plotting = Plotting::Unset; |
| |
| let mut reports: Vec<Box<Report>> = vec![]; |
| reports.push(Box::new(CliReport::new(false, false, false))); |
| reports.push(Box::new(FileCsvReport)); |
| |
| let output_directory = |
| match std::env::vars().find(|&(ref key, _)| key == "CARGO_TARGET_DIR") { |
| Some((_, value)) => format!("{}/criterion", value), |
| None => "target/criterion".to_owned(), |
| }; |
| |
| Criterion { |
| config: BenchmarkConfig { |
| confidence_level: 0.95, |
| measurement_time: Duration::new(5, 0), |
| noise_threshold: 0.01, |
| nresamples: 100_000, |
| sample_size: 100, |
| significance_level: 0.05, |
| warm_up_time: Duration::new(3, 0), |
| }, |
| plotting, |
| filter: None, |
| report: Box::new(Reports::new(reports)), |
| baseline_directory: "base".to_owned(), |
| baseline: Baseline::Save, |
| profile_time: None, |
| test_mode: false, |
| list_mode: false, |
| output_directory, |
| } |
| } |
| } |
| |
| impl Criterion { |
| /// Changes the default size of the sample for benchmarks run with this runner. |
| /// |
| /// A bigger sample should yield more accurate results if paired with a sufficiently large |
| /// measurement time. |
| /// |
| /// Sample size must be at least 2. |
| /// |
| /// # Panics |
| /// |
| /// Panics if set to zero or one |
| pub fn sample_size(mut self, n: usize) -> Criterion { |
| assert!(n >= 2); |
| if n < 10 { |
| println!("Warning: Sample sizes < 10 will be disallowed in Criterion.rs 0.3.0."); |
| } |
| |
| self.config.sample_size = n; |
| self |
| } |
| |
| /// Changes the default warm up time for benchmarks run with this runner. |
| /// |
| /// # Panics |
| /// |
| /// Panics if the input duration is zero |
| pub fn warm_up_time(mut self, dur: Duration) -> Criterion { |
| assert!(dur.to_nanos() > 0); |
| |
| self.config.warm_up_time = dur; |
| self |
| } |
| |
| /// Changes the default measurement time for benchmarks run with this runner. |
| /// |
| /// With a longer time, the measurement will become more resilient to transitory peak loads |
| /// caused by external programs |
| /// |
| /// **Note**: If the measurement time is too "low", Criterion will automatically increase it |
| /// |
| /// # Panics |
| /// |
| /// Panics if the input duration in zero |
| pub fn measurement_time(mut self, dur: Duration) -> Criterion { |
| assert!(dur.to_nanos() > 0); |
| |
| self.config.measurement_time = dur; |
| self |
| } |
| |
| /// Changes the default number of resamples for benchmarks run with this runner. |
| /// |
| /// Number of resamples to use for the |
| /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling) |
| /// |
| /// A larger number of resamples reduces the random sampling errors, which are inherent to the |
| /// bootstrap method, but also increases the analysis time |
| /// |
| /// # Panics |
| /// |
| /// Panics if the number of resamples is set to zero |
| pub fn nresamples(mut self, n: usize) -> Criterion { |
| assert!(n > 0); |
| |
| self.config.nresamples = n; |
| self |
| } |
| |
| /// Changes the default noise threshold for benchmarks run with this runner. |
| /// |
| /// This threshold is used to decide if an increase of `X%` in the execution time is considered |
| /// significant or should be flagged as noise |
| /// |
| /// *Note:* A value of `0.02` is equivalent to `2%` |
| /// |
| /// # Panics |
| /// |
| /// Panics is the threshold is set to a negative value |
| pub fn noise_threshold(mut self, threshold: f64) -> Criterion { |
| assert!(threshold >= 0.0); |
| |
| self.config.noise_threshold = threshold; |
| self |
| } |
| |
| /// Changes the default confidence level for benchmarks run with this runner |
| /// |
| /// The confidence level is used to calculate the |
| /// [confidence intervals](https://en.wikipedia.org/wiki/Confidence_interval) of the estimated |
| /// statistics |
| /// |
| /// # Panics |
| /// |
| /// Panics if the confidence level is set to a value outside the `(0, 1)` range |
| pub fn confidence_level(mut self, cl: f64) -> Criterion { |
| assert!(cl > 0.0 && cl < 1.0); |
| |
| self.config.confidence_level = cl; |
| self |
| } |
| |
| /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance) |
| /// for benchmarks run with this runner |
| /// |
| /// The significance level is used for |
| /// [hypothesis testing](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) |
| /// |
| /// # Panics |
| /// |
| /// Panics if the significance level is set to a value outside the `(0, 1)` range |
| pub fn significance_level(mut self, sl: f64) -> Criterion { |
| assert!(sl > 0.0 && sl < 1.0); |
| |
| self.config.significance_level = sl; |
| self |
| } |
| |
| /// Enables plotting |
| #[cfg(feature = "html_reports")] |
| pub fn with_plots(mut self) -> Criterion { |
| use criterion_plot::VersionError; |
| self.plotting = match criterion_plot::version() { |
| Ok(_) => { |
| let mut reports: Vec<Box<Report>> = vec![]; |
| reports.push(Box::new(CliReport::new(false, false, false))); |
| reports.push(Box::new(FileCsvReport)); |
| reports.push(Box::new(Html::new())); |
| self.report = Box::new(Reports::new(reports)); |
| Plotting::Enabled |
| } |
| Err(e) => { |
| match e { |
| VersionError::Exec(_) => println!("Gnuplot not found, disabling plotting"), |
| e => println!("Gnuplot not found or not usable, disabling plotting\n{}", e), |
| } |
| Plotting::NotAvailable |
| } |
| }; |
| |
| self |
| } |
| |
| /// Enables plotting |
| #[cfg(not(feature = "html_reports"))] |
| pub fn with_plots(self) -> Criterion { |
| self |
| } |
| |
| /// Disables plotting |
| pub fn without_plots(mut self) -> Criterion { |
| self.plotting = Plotting::Disabled; |
| self |
| } |
| |
| /// Return true if generation of the plots is possible. |
| #[cfg(feature = "html_reports")] |
| pub fn can_plot(&self) -> bool { |
| match self.plotting { |
| Plotting::NotAvailable => false, |
| Plotting::Enabled => true, |
| _ => criterion_plot::version().is_ok(), |
| } |
| } |
| |
| /// Return true if generation of the plots is possible. |
| #[cfg(not(feature = "html_reports"))] |
| pub fn can_plot(&self) -> bool { |
| false |
| } |
| |
| /// Names an explicit baseline and enables overwriting the previous results. |
| pub fn save_baseline(mut self, baseline: String) -> Criterion { |
| self.baseline_directory = baseline; |
| self.baseline = Baseline::Save; |
| self |
| } |
| |
| /// Names an explicit baseline and disables overwriting the previous results. |
| pub fn retain_baseline(mut self, baseline: String) -> Criterion { |
| self.baseline_directory = baseline; |
| self.baseline = Baseline::Compare; |
| self |
| } |
| |
| /// Filters the benchmarks. Only benchmarks with names that contain the |
| /// given string will be executed. |
| pub fn with_filter<S: Into<String>>(mut self, filter: S) -> Criterion { |
| self.filter = Some(filter.into()); |
| |
| self |
| } |
| |
| /// Set the output directory (currently for testing only) |
| #[doc(hidden)] |
| pub fn output_directory(mut self, path: &std::path::Path) -> Criterion { |
| self.output_directory = path.to_string_lossy().into_owned(); |
| |
| self |
| } |
| |
| /// Generate the final summary at the end of a run. |
| #[doc(hidden)] |
| pub fn final_summary(&self) { |
| if self.profile_time.is_some() || self.test_mode { |
| return; |
| } |
| |
| let report_context = ReportContext { |
| output_directory: self.output_directory.clone(), |
| plotting: self.plotting, |
| plot_config: PlotConfiguration::default(), |
| test_mode: self.test_mode, |
| }; |
| |
| self.report.final_summary(&report_context); |
| } |
| |
| /// Configure this criterion struct based on the command-line arguments to |
| /// this process. |
| pub fn configure_from_args(mut self) -> Criterion { |
| use clap::{App, Arg}; |
| let matches = App::new("Criterion Benchmark") |
| .arg(Arg::with_name("FILTER") |
| .help("Skip benchmarks whose names do not contain FILTER.") |
| .index(1)) |
| .arg(Arg::with_name("color") |
| .short("c") |
| .long("color") |
| .alias("colour") |
| .takes_value(true) |
| .possible_values(&["auto", "always", "never"]) |
| .default_value("auto") |
| .help("Configure coloring of output. always = always colorize output, never = never colorize output, auto = colorize output if output is a tty and compiled for unix.")) |
| .arg(Arg::with_name("verbose") |
| .short("v") |
| .long("verbose") |
| .help("Print additional statistical information.")) |
| .arg(Arg::with_name("noplot") |
| .short("n") |
| .long("noplot") |
| .help("Disable plot and HTML generation.")) |
| .arg(Arg::with_name("save-baseline") |
| .short("s") |
| .long("save-baseline") |
| .default_value("base") |
| .help("Save results under a named baseline.")) |
| .arg(Arg::with_name("baseline") |
| .short("b") |
| .long("baseline") |
| .takes_value(true) |
| .conflicts_with("save-baseline") |
| .help("Compare to a named baseline.")) |
| .arg(Arg::with_name("list") |
| .long("list") |
| .help("List all benchmarks")) |
| .arg(Arg::with_name("measure-only") |
| .long("measure-only") |
| .hidden(true) |
| .help("Only perform measurements; do no analysis or storage of results. This is useful eg. when profiling the benchmarks, to reduce clutter in the profiling data.")) |
| .arg(Arg::with_name("profile-time") |
| .long("profile-time") |
| .takes_value(true) |
| .help("Iterate each benchmark for approximately the given number of seconds, doing no analysis and without storing the results. Useful for running the benchmarks in a profiler.")) |
| .arg(Arg::with_name("test") |
| .long("test") |
| .help("Run the benchmarks once, to verify that they execute successfully, but do not measure or report the results.")) |
| //Ignored but always passed to benchmark executables |
| .arg(Arg::with_name("bench") |
| .hidden(true) |
| .long("bench")) |
| .arg(Arg::with_name("version") |
| .hidden(true) |
| .short("V") |
| .long("version")) |
| .after_help(" |
| This executable is a Criterion.rs benchmark. |
| See https://github.com/bheisler/criterion.rs for more details. |
| |
| To enable debug output, define the environment variable CRITERION_DEBUG. |
| Criterion.rs will output more debug information and will save the gnuplot |
| scripts alongside the generated plots. |
| ") |
| .get_matches(); |
| |
| if let Some(filter) = matches.value_of("FILTER") { |
| self = self.with_filter(filter); |
| } |
| |
| let verbose = matches.is_present("verbose"); |
| let stdout_isatty = atty::is(atty::Stream::Stdout); |
| let mut enable_text_overwrite = stdout_isatty && !verbose && !debug_enabled(); |
| let enable_text_coloring; |
| match matches.value_of("color") { |
| Some("always") => { |
| enable_text_coloring = true; |
| } |
| Some("never") => { |
| enable_text_coloring = false; |
| enable_text_overwrite = false; |
| } |
| _ => enable_text_coloring = stdout_isatty, |
| } |
| |
| if matches.is_present("noplot") || matches.is_present("test") { |
| self = self.without_plots(); |
| } else { |
| self = self.with_plots(); |
| } |
| |
| if let Some(dir) = matches.value_of("save-baseline") { |
| self.baseline = Baseline::Save; |
| self.baseline_directory = dir.to_owned() |
| } |
| if let Some(dir) = matches.value_of("baseline") { |
| self.baseline = Baseline::Compare; |
| self.baseline_directory = dir.to_owned(); |
| } |
| |
| let mut reports: Vec<Box<Report>> = vec![]; |
| reports.push(Box::new(CliReport::new( |
| enable_text_overwrite, |
| enable_text_coloring, |
| verbose, |
| ))); |
| reports.push(Box::new(FileCsvReport)); |
| |
| // TODO: Remove this in 0.3.0 |
| if matches.is_present("measure-only") { |
| println!("Warning: The '--measure-only' argument is deprecated and will be removed in Criterion.rs 0.3.0. Use '--profile-time' instead."); |
| self.profile_time = Some(Duration::from_secs(5)); |
| } |
| if matches.is_present("profile-time") { |
| let num_seconds = value_t!(matches.value_of("profile-time"), u64).unwrap_or_else(|e| { |
| println!("{}", e); |
| std::process::exit(1) |
| }); |
| |
| if num_seconds < 1 { |
| println!("Profile time must be at least one second."); |
| std::process::exit(1); |
| } |
| |
| self.profile_time = Some(Duration::from_secs(num_seconds)); |
| } |
| self.test_mode = matches.is_present("test"); |
| if matches.is_present("list") { |
| self.test_mode = true; |
| self.list_mode = true; |
| } |
| |
| #[cfg(feature = "html_reports")] |
| { |
| if self.profile_time.is_none() { |
| reports.push(Box::new(Html::new())); |
| } |
| } |
| |
| self.report = Box::new(Reports::new(reports)); |
| |
| self |
| } |
| |
| fn filter_matches(&self, id: &str) -> bool { |
| match self.filter { |
| Some(ref string) => id.contains(string), |
| None => true, |
| } |
| } |
| |
| /// Benchmarks a function |
| /// |
| /// # Example |
| /// |
| /// ```rust |
| /// # #[macro_use] extern crate criterion; |
| /// # use self::criterion::*; |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// // Setup (construct data, allocate memory, etc) |
| /// c.bench_function( |
| /// "function_name", |
| /// |b| b.iter(|| { |
| /// // Code to benchmark goes here |
| /// }), |
| /// ); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| pub fn bench_function<F>(&mut self, id: &str, f: F) -> &mut Criterion |
| where |
| F: FnMut(&mut Bencher) + 'static, |
| { |
| self.bench(id, Benchmark::new(id, f)) |
| } |
| |
| /// Benchmarks multiple functions |
| /// |
| /// All functions get the same input and are compared with the other implementations. |
| /// Works similar to `bench_function`, but with multiple functions. |
| /// |
| /// # Example |
| /// |
| /// ``` rust |
| /// # #[macro_use] extern crate criterion; |
| /// # use self::criterion::*; |
| /// # fn seq_fib(i: &u32) {} |
| /// # fn par_fib(i: &u32) {} |
| /// |
| /// fn bench_seq_fib(b: &mut Bencher, i: &u32) { |
| /// b.iter(|| { |
| /// seq_fib(i); |
| /// }); |
| /// } |
| /// |
| /// fn bench_par_fib(b: &mut Bencher, i: &u32) { |
| /// b.iter(|| { |
| /// par_fib(i); |
| /// }); |
| /// } |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// let sequential_fib = Fun::new("Sequential", bench_seq_fib); |
| /// let parallel_fib = Fun::new("Parallel", bench_par_fib); |
| /// let funs = vec![sequential_fib, parallel_fib]; |
| /// |
| /// c.bench_functions("Fibonacci", funs, 14); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| pub fn bench_functions<I>(&mut self, id: &str, funs: Vec<Fun<I>>, input: I) -> &mut Criterion |
| where |
| I: fmt::Debug + 'static, |
| { |
| let benchmark = ParameterizedBenchmark::with_functions( |
| funs.into_iter().map(|fun| fun.f).collect(), |
| vec![input], |
| ); |
| |
| self.bench(id, benchmark) |
| } |
| |
| /// Benchmarks a function under various inputs |
| /// |
| /// This is a convenience method to execute several related benchmarks. Each benchmark will |
| /// receive the id: `${id}/${input}`. |
| /// |
| /// # Example |
| /// |
| /// ```rust |
| /// # #[macro_use] extern crate criterion; |
| /// # use self::criterion::*; |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// c.bench_function_over_inputs("from_elem", |
| /// |b: &mut Bencher, size: &usize| { |
| /// b.iter(|| vec![0u8; *size]); |
| /// }, |
| /// vec![1024, 2048, 4096] |
| /// ); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| pub fn bench_function_over_inputs<I, F>(&mut self, id: &str, f: F, inputs: I) -> &mut Criterion |
| where |
| I: IntoIterator, |
| I::Item: fmt::Debug + 'static, |
| F: FnMut(&mut Bencher, &I::Item) + 'static, |
| { |
| self.bench(id, ParameterizedBenchmark::new(id, f, inputs)) |
| } |
| |
| /// Benchmarks an external program |
| /// |
| /// The external program must: |
| /// |
| /// * Read the number of iterations from stdin |
| /// * Execute the routine to benchmark that many times |
| /// * Print the elapsed time (in nanoseconds) to stdout |
| /// |
| /// ```rust,no_run |
| /// # use std::io::{self, BufRead}; |
| /// # use std::time::Instant; |
| /// # use std::time::Duration; |
| /// # trait DurationExt { fn to_nanos(&self) -> u64 { 0 } } |
| /// # impl DurationExt for Duration {} |
| /// // Example of an external program that implements this protocol |
| /// |
| /// fn main() { |
| /// let stdin = io::stdin(); |
| /// let ref mut stdin = stdin.lock(); |
| /// |
| /// // For each line in stdin |
| /// for line in stdin.lines() { |
| /// // Parse line as the number of iterations |
| /// let iters: u64 = line.unwrap().trim().parse().unwrap(); |
| /// |
| /// // Setup |
| /// |
| /// // Benchmark |
| /// let start = Instant::now(); |
| /// // Execute the routine "iters" times |
| /// for _ in 0..iters { |
| /// // Code to benchmark goes here |
| /// } |
| /// let elapsed = start.elapsed(); |
| /// |
| /// // Teardown |
| /// |
| /// // Report elapsed time in nanoseconds to stdout |
| /// println!("{}", elapsed.to_nanos()); |
| /// } |
| /// } |
| /// ``` |
| #[deprecated( |
| since = "0.2.6", |
| note = "External program benchmarks were rarely used and are awkward to maintain, so they are scheduled for deletion in 0.3.0" |
| )] |
| #[allow(deprecated)] |
| pub fn bench_program(&mut self, id: &str, program: Command) -> &mut Criterion { |
| self.bench(id, Benchmark::new_external(id, program)) |
| } |
| |
| /// Benchmarks an external program under various inputs |
| /// |
| /// This is a convenience method to execute several related benchmarks. Each benchmark will |
| /// receive the id: `${id}/${input}`. |
| #[deprecated( |
| since = "0.2.6", |
| note = "External program benchmarks were rarely used and are awkward to maintain, so they are scheduled for deletion in 0.3.0" |
| )] |
| #[allow(deprecated)] |
| pub fn bench_program_over_inputs<I, F>( |
| &mut self, |
| id: &str, |
| mut program: F, |
| inputs: I, |
| ) -> &mut Criterion |
| where |
| F: FnMut() -> Command + 'static, |
| I: IntoIterator, |
| I::Item: fmt::Debug + 'static, |
| { |
| self.bench( |
| id, |
| ParameterizedBenchmark::new_external( |
| id, |
| move |i| { |
| let mut command = program(); |
| command.arg(format!("{:?}", i)); |
| command |
| }, |
| inputs, |
| ), |
| ) |
| } |
| |
| /// Executes the given benchmark. Use this variant to execute benchmarks |
| /// with complex configuration. This can be used to compare multiple |
| /// functions, execute benchmarks with custom configuration settings and |
| /// more. See the Benchmark and ParameterizedBenchmark structs for more |
| /// information. |
| /// |
| /// ```rust |
| /// # #[macro_use] extern crate criterion; |
| /// # use criterion::*; |
| /// # fn routine_1() {} |
| /// # fn routine_2() {} |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// // Setup (construct data, allocate memory, etc) |
| /// c.bench( |
| /// "routines", |
| /// Benchmark::new("routine_1", |b| b.iter(|| routine_1())) |
| /// .with_function("routine_2", |b| b.iter(|| routine_2())) |
| /// .sample_size(50) |
| /// ); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| pub fn bench<B: BenchmarkDefinition>( |
| &mut self, |
| group_id: &str, |
| benchmark: B, |
| ) -> &mut Criterion { |
| benchmark.run(group_id, self); |
| self |
| } |
| } |
| |
| mod plotting { |
| #[derive(Debug, Clone, Copy)] |
| pub enum Plotting { |
| Unset, |
| Disabled, |
| Enabled, |
| NotAvailable, |
| } |
| |
| impl Plotting { |
| pub fn is_enabled(self) -> bool { |
| match self { |
| Plotting::Enabled => true, |
| _ => false, |
| } |
| } |
| } |
| } |
| |
| trait DurationExt { |
| fn to_nanos(&self) -> u64; |
| } |
| |
| const NANOS_PER_SEC: u64 = 1_000_000_000; |
| |
| impl DurationExt for Duration { |
| fn to_nanos(&self) -> u64 { |
| self.as_secs() * NANOS_PER_SEC + u64::from(self.subsec_nanos()) |
| } |
| } |
| |
| #[derive(Clone, Copy, PartialEq, Deserialize, Serialize, Debug)] |
| struct ConfidenceInterval { |
| confidence_level: f64, |
| lower_bound: f64, |
| upper_bound: f64, |
| } |
| |
| #[derive(Clone, Copy, PartialEq, Deserialize, Serialize, Debug)] |
| struct Estimate { |
| /// The confidence interval for this estimate |
| confidence_interval: ConfidenceInterval, |
| /// |
| point_estimate: f64, |
| /// The standard error of this estimate |
| standard_error: f64, |
| } |
| |
| fn build_estimates( |
| distributions: &Distributions, |
| points: &BTreeMap<Statistic, f64>, |
| cl: f64, |
| ) -> Estimates { |
| distributions |
| .iter() |
| .map(|(&statistic, distribution)| { |
| let point_estimate = points[&statistic]; |
| let (lb, ub) = distribution.confidence_interval(cl); |
| |
| ( |
| statistic, |
| Estimate { |
| confidence_interval: ConfidenceInterval { |
| confidence_level: cl, |
| lower_bound: lb, |
| upper_bound: ub, |
| }, |
| point_estimate, |
| standard_error: distribution.std_dev(None), |
| }, |
| ) |
| }) |
| .collect() |
| } |
| |
| /// Enum representing different ways of measuring the throughput of benchmarked code. |
| /// If the throughput setting is configured for a benchmark then the estimated throughput will |
| /// be reported as well as the time per iteration. |
| #[derive(Debug, Clone, Serialize, Deserialize)] |
| pub enum Throughput { |
| /// Measure throughput in terms of bytes/second. The value should be the number of bytes |
| /// processed by one iteration of the benchmarked code. Typically, this would be the length of |
| /// an input string or `&[u8]`. |
| Bytes(u32), |
| |
| /// Measure throughput in terms of elements/second. The value should be the number of elements |
| /// processed by one iteration of the benchmarked code. Typically, this would be the size of a |
| /// collection, but could also be the number of lines of input text or the number of values to |
| /// parse. |
| Elements(u32), |
| } |
| |
| /// Axis scaling type |
| #[derive(Debug, Clone, Copy)] |
| pub enum AxisScale { |
| /// Axes scale linearly |
| Linear, |
| |
| /// Axes scale logarithmically |
| Logarithmic, |
| } |
| |
| /// Contains the configuration options for the plots generated by a particular benchmark |
| /// or benchmark group. |
| /// |
| /// ```rust |
| /// use self::criterion::{Bencher, Criterion, Benchmark, PlotConfiguration, AxisScale}; |
| /// |
| /// let plot_config = PlotConfiguration::default() |
| /// .summary_scale(AxisScale::Logarithmic); |
| /// |
| /// Benchmark::new("test", |b| b.iter(|| 10)) |
| /// .plot_config(plot_config); |
| /// ``` |
| #[derive(Debug, Clone)] |
| pub struct PlotConfiguration { |
| summary_scale: AxisScale, |
| } |
| |
| impl Default for PlotConfiguration { |
| fn default() -> PlotConfiguration { |
| PlotConfiguration { |
| summary_scale: AxisScale::Linear, |
| } |
| } |
| } |
| |
| impl PlotConfiguration { |
| /// Set the axis scale (linear or logarithmic) for the summary plots. Typically, you would |
| /// set this to logarithmic if benchmarking over a range of inputs which scale exponentially. |
| /// Defaults to linear. |
| pub fn summary_scale(mut self, new_scale: AxisScale) -> PlotConfiguration { |
| self.summary_scale = new_scale; |
| self |
| } |
| } |
