| use crate::analysis; |
| use crate::connection::OutgoingMessage; |
| use crate::measurement::{Measurement, WallTime}; |
| use crate::report::{BenchmarkId, ReportContext}; |
| use crate::routine::{Function, Routine}; |
| use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput}; |
| use std::cell::RefCell; |
| use std::fmt::Debug; |
| use std::marker::Sized; |
| use std::time::Duration; |
| |
| // TODO: Move the benchmark config stuff to a separate module for easier use. |
| |
| /// Struct containing all of the configuration options for a benchmark. |
| pub struct BenchmarkConfig { |
| pub confidence_level: f64, |
| pub measurement_time: Duration, |
| pub noise_threshold: f64, |
| pub nresamples: usize, |
| pub sample_size: usize, |
| pub significance_level: f64, |
| pub warm_up_time: Duration, |
| pub sampling_mode: SamplingMode, |
| } |
| |
| /// Struct representing a partially-complete per-benchmark configuration. |
| #[derive(Clone)] |
| pub(crate) struct PartialBenchmarkConfig { |
| pub(crate) confidence_level: Option<f64>, |
| pub(crate) measurement_time: Option<Duration>, |
| pub(crate) noise_threshold: Option<f64>, |
| pub(crate) nresamples: Option<usize>, |
| pub(crate) sample_size: Option<usize>, |
| pub(crate) significance_level: Option<f64>, |
| pub(crate) warm_up_time: Option<Duration>, |
| pub(crate) sampling_mode: Option<SamplingMode>, |
| pub(crate) plot_config: PlotConfiguration, |
| } |
| |
| impl Default for PartialBenchmarkConfig { |
| fn default() -> Self { |
| PartialBenchmarkConfig { |
| confidence_level: None, |
| measurement_time: None, |
| noise_threshold: None, |
| nresamples: None, |
| sample_size: None, |
| significance_level: None, |
| warm_up_time: None, |
| plot_config: PlotConfiguration::default(), |
| sampling_mode: None, |
| } |
| } |
| } |
| |
| impl PartialBenchmarkConfig { |
| pub(crate) fn to_complete(&self, defaults: &BenchmarkConfig) -> BenchmarkConfig { |
| BenchmarkConfig { |
| confidence_level: self.confidence_level.unwrap_or(defaults.confidence_level), |
| measurement_time: self.measurement_time.unwrap_or(defaults.measurement_time), |
| noise_threshold: self.noise_threshold.unwrap_or(defaults.noise_threshold), |
| nresamples: self.nresamples.unwrap_or(defaults.nresamples), |
| sample_size: self.sample_size.unwrap_or(defaults.sample_size), |
| significance_level: self |
| .significance_level |
| .unwrap_or(defaults.significance_level), |
| warm_up_time: self.warm_up_time.unwrap_or(defaults.warm_up_time), |
| sampling_mode: self.sampling_mode.unwrap_or(defaults.sampling_mode), |
| } |
| } |
| } |
| |
| pub(crate) struct NamedRoutine<T, M: Measurement = WallTime> { |
| pub id: String, |
| pub(crate) f: Box<RefCell<dyn Routine<M, T>>>, |
| } |
| |
| /// Structure representing a benchmark (or group of benchmarks) |
| /// which take one parameter. |
| #[doc(hidden)] |
| pub struct ParameterizedBenchmark<T: Debug, M: Measurement = WallTime> { |
| config: PartialBenchmarkConfig, |
| values: Vec<T>, |
| routines: Vec<NamedRoutine<T, M>>, |
| throughput: Option<Box<dyn Fn(&T) -> Throughput>>, |
| } |
| |
| /// Structure representing a benchmark (or group of benchmarks) |
| /// which takes no parameters. |
| #[doc(hidden)] |
| pub struct Benchmark<M: Measurement = WallTime> { |
| config: PartialBenchmarkConfig, |
| routines: Vec<NamedRoutine<(), M>>, |
| throughput: Option<Throughput>, |
| } |
| |
| /// Common trait for `Benchmark` and `ParameterizedBenchmark`. Not intended to be |
| /// used outside of Criterion.rs. |
| #[doc(hidden)] |
| pub trait BenchmarkDefinition<M: Measurement = WallTime>: Sized { |
| #[doc(hidden)] |
| fn run(self, group_id: &str, c: &mut Criterion<M>); |
| } |
| |
| macro_rules! benchmark_config { |
| ($type:tt) => { |
| /// Changes the size of the sample for this benchmark |
| /// |
| /// A bigger sample should yield more accurate results if paired with a sufficiently large |
| /// measurement time. |
| /// |
| /// Sample size must be at least 10. |
| /// |
| /// # Panics |
| /// |
| /// Panics if n < 10. |
| pub fn sample_size(mut self, n: usize) -> Self { |
| assert!(n >= 10); |
| |
| self.config.sample_size = Some(n); |
| self |
| } |
| |
| /// Changes the warm up time for this benchmark |
| /// |
| /// # Panics |
| /// |
| /// Panics if the input duration is zero |
| pub fn warm_up_time(mut self, dur: Duration) -> Self { |
| assert!(dur.to_nanos() > 0); |
| |
| self.config.warm_up_time = Some(dur); |
| self |
| } |
| |
| /// Changes the target measurement time for this benchmark. Criterion will attempt |
| /// to spent approximately this amount of time measuring the benchmark. |
| /// With a longer time, the measurement will become more resilient to transitory peak loads |
| /// caused by external programs. |
| /// |
| /// # Panics |
| /// |
| /// Panics if the input duration in zero |
| pub fn measurement_time(mut self, dur: Duration) -> Self { |
| assert!(dur.to_nanos() > 0); |
| |
| self.config.measurement_time = Some(dur); |
| self |
| } |
| |
| /// Changes the number of resamples for this benchmark |
| /// |
| /// Number of resamples to use for the |
| /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling) |
| /// |
| /// A larger number of resamples reduces the random sampling errors, which are inherent to the |
| /// bootstrap method, but also increases the analysis time. |
| /// |
| /// # Panics |
| /// |
| /// Panics if the number of resamples is set to zero |
| pub fn nresamples(mut self, n: usize) -> Self { |
| assert!(n > 0); |
| if n <= 1000 { |
| println!("\nWarning: It is not recommended to reduce nresamples below 1000."); |
| } |
| |
| self.config.nresamples = Some(n); |
| self |
| } |
| |
| /// Changes the default noise threshold for this benchmark. The noise threshold |
| /// is used to filter out small changes in performance, even if they are statistically |
| /// significant. Sometimes benchmarking the same code twice will result in small but |
| /// statistically significant differences solely because of noise. This provides a way to filter |
| /// out some of these false positives at the cost of making it harder to detect small changes |
| /// to the true performance of the benchmark. |
| /// |
| /// The default is 0.01, meaning that changes smaller than 1% will be ignored. |
| /// |
| /// # Panics |
| /// |
| /// Panics if the threshold is set to a negative value |
| pub fn noise_threshold(mut self, threshold: f64) -> Self { |
| assert!(threshold >= 0.0); |
| |
| self.config.noise_threshold = Some(threshold); |
| self |
| } |
| |
| /// Changes the default confidence level for this benchmark. The confidence |
| /// level is the desired probability that the true runtime lies within the estimated |
| /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is |
| /// 0.95, meaning that the confidence interval should capture the true value 95% of the time. |
| /// |
| /// # Panics |
| /// |
| /// Panics if the confidence level is set to a value outside the `(0, 1)` range |
| pub fn confidence_level(mut self, cl: f64) -> Self { |
| assert!(cl > 0.0 && cl < 1.0); |
| if cl < 0.5 { |
| println!("\nWarning: It is not recommended to reduce confidence level below 0.5."); |
| } |
| |
| self.config.confidence_level = Some(cl); |
| self |
| } |
| |
| /// Changes the default [significance level](https://en.wikipedia.org/wiki/Statistical_significance) |
| /// for this benchmark. This is used to perform a |
| /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if |
| /// the measurements from this run are different from the measured performance of the last run. |
| /// The significance level is the desired probability that two measurements of identical code |
| /// will be considered 'different' due to noise in the measurements. The default value is 0.05, |
| /// meaning that approximately 5% of identical benchmarks will register as different due to |
| /// noise. |
| /// |
| /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase |
| /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to |
| /// detect small but real changes in the performance. By setting the significance level |
| /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also |
| /// report more spurious differences. |
| /// |
| /// See also the noise threshold setting. |
| /// |
| /// # Panics |
| /// |
| /// Panics if the significance level is set to a value outside the `(0, 1)` range |
| pub fn significance_level(mut self, sl: f64) -> Self { |
| assert!(sl > 0.0 && sl < 1.0); |
| |
| self.config.significance_level = Some(sl); |
| self |
| } |
| |
| /// Changes the plot configuration for this benchmark. |
| pub fn plot_config(mut self, new_config: PlotConfiguration) -> Self { |
| self.config.plot_config = new_config; |
| self |
| } |
| |
| /// Changes the sampling mode for this benchmark. |
| pub fn sampling_mode(mut self, new_mode: SamplingMode) -> Self { |
| self.config.sampling_mode = Some(new_mode); |
| self |
| } |
| }; |
| } |
| |
| impl<M> Benchmark<M> |
| where |
| M: Measurement + 'static, |
| { |
| benchmark_config!(Benchmark); |
| |
| /// Create a new benchmark group and adds the given function to it. |
| /// |
| /// # Example |
| /// |
| /// ```rust |
| /// # #[macro_use] extern crate criterion; |
| /// # use criterion::*; |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// // One-time setup goes here |
| /// c.bench( |
| /// "my_group", |
| /// Benchmark::new("my_function", |b| b.iter(|| { |
| /// // Code to benchmark goes here |
| /// })), |
| /// ); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| pub fn new<S, F>(id: S, f: F) -> Benchmark<M> |
| where |
| S: Into<String>, |
| F: FnMut(&mut Bencher<'_, M>) + 'static, |
| { |
| Benchmark { |
| config: PartialBenchmarkConfig::default(), |
| routines: vec![], |
| throughput: None, |
| } |
| .with_function(id, f) |
| } |
| |
| /// Add a function to the benchmark group. |
| pub fn with_function<S, F>(mut self, id: S, mut f: F) -> Benchmark<M> |
| where |
| S: Into<String>, |
| F: FnMut(&mut Bencher<'_, M>) + 'static, |
| { |
| let routine = NamedRoutine { |
| id: id.into(), |
| f: Box::new(RefCell::new(Function::new(move |b, _| f(b)))), |
| }; |
| self.routines.push(routine); |
| self |
| } |
| |
| /// Set the input size for this benchmark group. Used for reporting the |
| /// throughput. |
| pub fn throughput(mut self, throughput: Throughput) -> Benchmark<M> { |
| self.throughput = Some(throughput); |
| self |
| } |
| } |
| |
| impl<M: Measurement> BenchmarkDefinition<M> for Benchmark<M> { |
| fn run(self, group_id: &str, c: &mut Criterion<M>) { |
| let report_context = ReportContext { |
| output_directory: c.output_directory.clone(), |
| plot_config: self.config.plot_config.clone(), |
| }; |
| |
| let config = self.config.to_complete(&c.config); |
| let num_routines = self.routines.len(); |
| |
| let mut all_ids = vec![]; |
| let mut any_matched = false; |
| |
| if let Some(conn) = &c.connection { |
| conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id }) |
| .unwrap(); |
| } |
| |
| for routine in self.routines { |
| let function_id = if num_routines == 1 && group_id == routine.id { |
| None |
| } else { |
| Some(routine.id) |
| }; |
| |
| let mut id = BenchmarkId::new( |
| group_id.to_owned(), |
| function_id, |
| None, |
| self.throughput.clone(), |
| ); |
| |
| id.ensure_directory_name_unique(&c.all_directories); |
| c.all_directories.insert(id.as_directory_name().to_owned()); |
| id.ensure_title_unique(&c.all_titles); |
| c.all_titles.insert(id.as_title().to_owned()); |
| |
| let do_run = c.filter_matches(id.id()); |
| any_matched |= do_run; |
| |
| execute_benchmark( |
| do_run, |
| &id, |
| c, |
| &config, |
| &mut *routine.f.borrow_mut(), |
| &report_context, |
| &(), |
| self.throughput.clone(), |
| ); |
| |
| all_ids.push(id); |
| } |
| |
| if let Some(conn) = &c.connection { |
| conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id }) |
| .unwrap(); |
| conn.serve_value_formatter(c.measurement.formatter()) |
| .unwrap(); |
| } |
| |
| if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() { |
| c.report |
| .summarize(&report_context, &all_ids, c.measurement.formatter()); |
| } |
| if any_matched { |
| c.report.group_separator(); |
| } |
| } |
| } |
| |
| impl<T, M> ParameterizedBenchmark<T, M> |
| where |
| T: Debug + 'static, |
| M: Measurement + 'static, |
| { |
| benchmark_config!(ParameterizedBenchmark); |
| |
| pub(crate) fn with_functions( |
| functions: Vec<NamedRoutine<T, M>>, |
| parameters: Vec<T>, |
| ) -> ParameterizedBenchmark<T, M> { |
| ParameterizedBenchmark { |
| config: PartialBenchmarkConfig::default(), |
| values: parameters, |
| routines: functions, |
| throughput: None, |
| } |
| } |
| |
| /// Create a new parameterized benchmark group and adds the given function |
| /// to it. |
| /// The function under test must follow the setup - bench - teardown pattern: |
| /// |
| /// # Example |
| /// |
| /// ```rust |
| /// # #[macro_use] extern crate criterion; |
| /// # use criterion::*; |
| /// |
| /// fn bench(c: &mut Criterion) { |
| /// let parameters = vec![1u64, 2u64, 3u64]; |
| /// |
| /// // One-time setup goes here |
| /// c.bench( |
| /// "my_group", |
| /// ParameterizedBenchmark::new( |
| /// "my_function", |
| /// |b, param| b.iter(|| { |
| /// // Code to benchmark using param goes here |
| /// }), |
| /// parameters |
| /// ) |
| /// ); |
| /// } |
| /// |
| /// criterion_group!(benches, bench); |
| /// criterion_main!(benches); |
| /// ``` |
| pub fn new<S, F, I>(id: S, f: F, parameters: I) -> ParameterizedBenchmark<T, M> |
| where |
| S: Into<String>, |
| F: FnMut(&mut Bencher<'_, M>, &T) + 'static, |
| I: IntoIterator<Item = T>, |
| { |
| ParameterizedBenchmark { |
| config: PartialBenchmarkConfig::default(), |
| values: parameters.into_iter().collect(), |
| routines: vec![], |
| throughput: None, |
| } |
| .with_function(id, f) |
| } |
| |
| /// Add a function to the benchmark group. |
| pub fn with_function<S, F>(mut self, id: S, f: F) -> ParameterizedBenchmark<T, M> |
| where |
| S: Into<String>, |
| F: FnMut(&mut Bencher<'_, M>, &T) + 'static, |
| { |
| let routine = NamedRoutine { |
| id: id.into(), |
| f: Box::new(RefCell::new(Function::new(f))), |
| }; |
| self.routines.push(routine); |
| self |
| } |
| |
| /// Use the given function to calculate the input size for a given input. |
| pub fn throughput<F>(mut self, throughput: F) -> ParameterizedBenchmark<T, M> |
| where |
| F: Fn(&T) -> Throughput + 'static, |
| { |
| self.throughput = Some(Box::new(throughput)); |
| self |
| } |
| } |
| impl<T, M> BenchmarkDefinition<M> for ParameterizedBenchmark<T, M> |
| where |
| T: Debug + 'static, |
| M: Measurement + 'static, |
| { |
| fn run(self, group_id: &str, c: &mut Criterion<M>) { |
| let report_context = ReportContext { |
| output_directory: c.output_directory.clone(), |
| plot_config: self.config.plot_config.clone(), |
| }; |
| |
| let config = self.config.to_complete(&c.config); |
| let num_parameters = self.values.len(); |
| let num_routines = self.routines.len(); |
| |
| let mut all_ids = vec![]; |
| let mut any_matched = false; |
| |
| if let Some(conn) = &c.connection { |
| conn.send(&OutgoingMessage::BeginningBenchmarkGroup { group: group_id }) |
| .unwrap(); |
| } |
| |
| for routine in self.routines { |
| for value in &self.values { |
| let function_id = if num_routines == 1 && group_id == routine.id { |
| None |
| } else { |
| Some(routine.id.clone()) |
| }; |
| |
| let value_str = if num_parameters == 1 { |
| None |
| } else { |
| Some(format!("{:?}", value)) |
| }; |
| |
| let throughput = self.throughput.as_ref().map(|func| func(value)); |
| let mut id = BenchmarkId::new( |
| group_id.to_owned(), |
| function_id, |
| value_str, |
| throughput.clone(), |
| ); |
| |
| id.ensure_directory_name_unique(&c.all_directories); |
| c.all_directories.insert(id.as_directory_name().to_owned()); |
| id.ensure_title_unique(&c.all_titles); |
| c.all_titles.insert(id.as_title().to_owned()); |
| |
| let do_run = c.filter_matches(id.id()); |
| any_matched |= do_run; |
| |
| execute_benchmark( |
| do_run, |
| &id, |
| c, |
| &config, |
| &mut *routine.f.borrow_mut(), |
| &report_context, |
| value, |
| throughput, |
| ); |
| |
| all_ids.push(id); |
| } |
| } |
| |
| if let Some(conn) = &c.connection { |
| conn.send(&OutgoingMessage::FinishedBenchmarkGroup { group: group_id }) |
| .unwrap(); |
| conn.serve_value_formatter(c.measurement.formatter()) |
| .unwrap(); |
| } |
| |
| if all_ids.len() > 1 && any_matched && c.mode.is_benchmark() { |
| c.report |
| .summarize(&report_context, &all_ids, c.measurement.formatter()); |
| } |
| if any_matched { |
| c.report.group_separator(); |
| } |
| } |
| } |
| |
| #[cfg_attr(feature = "cargo-clippy", allow(clippy::too_many_arguments))] |
| fn execute_benchmark<T, M>( |
| do_run: bool, |
| id: &BenchmarkId, |
| c: &Criterion<M>, |
| config: &BenchmarkConfig, |
| routine: &mut dyn Routine<M, T>, |
| report_context: &ReportContext, |
| parameter: &T, |
| throughput: Option<Throughput>, |
| ) where |
| T: Debug, |
| M: Measurement, |
| { |
| match c.mode { |
| Mode::Benchmark => { |
| if let Some(conn) = &c.connection { |
| if do_run { |
| conn.send(&OutgoingMessage::BeginningBenchmark { id: id.into() }) |
| .unwrap(); |
| } else { |
| conn.send(&OutgoingMessage::SkippingBenchmark { id: id.into() }) |
| .unwrap(); |
| } |
| } |
| |
| if do_run { |
| analysis::common( |
| id, |
| routine, |
| config, |
| c, |
| report_context, |
| parameter, |
| throughput, |
| ); |
| } |
| } |
| Mode::List => { |
| if do_run { |
| println!("{}: bench", id); |
| } |
| } |
| Mode::Test => { |
| if do_run { |
| // In test mode, run the benchmark exactly once, then exit. |
| c.report.test_start(id, report_context); |
| routine.test(&c.measurement, parameter); |
| c.report.test_pass(id, report_context); |
| } |
| } |
| Mode::Profile(duration) => { |
| if do_run { |
| routine.profile(&c.measurement, id, c, report_context, duration, parameter); |
| } |
| } |
| } |
| } |
