Snap for 7280565 from d9e2661bbdf97b23123613e9b13b321162baeb1c to sc-release
Change-Id: Idd5a936bf0b942cb8fc700573bc5905b8a7f947a
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json
index 931fc83..b30c393 100644
--- a/.cargo_vcs_info.json
+++ b/.cargo_vcs_info.json
@@ -1,5 +1,5 @@
{
"git": {
- "sha1": "3fcbcd237e14306d102f2ea2f1285cd6285e086c"
+ "sha1": "1bf6f4badbddd98fc7b667d310605c5a204bc8e4"
}
}
diff --git a/.travis.yml b/.travis.yml
index 97755ef..d67d8fa 100755
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,7 +29,7 @@
env: DOCS=yes
- os: linux
env: GNUPLOT=yes
- rust: 1.36.0
+ rust: 1.39.0
addons:
apt:
packages:
@@ -62,6 +62,16 @@
export PATH=$HOME/.local/bin:$PATH;
fi
+before_cache:
+- find ./target/debug -maxdepth 1 -type f -delete
+- rm -rf ./target/debug/deps/criterion*
+- rm -rf ./target/debug/deps/bench*
+- rm -rf ./target/debug/.fingerprint/criterion*
+- rm -rf ./target/debug/.fingerprint/bench*
+- rm -f ./target/.rustc_info.json
+- rm -rf ./target/criterion
+- rm -rf ~/.cargo/registry/index/
+
install:
- sh ci/install.sh
diff --git a/Android.bp b/Android.bp
index 67e71fa..5338905 100644
--- a/Android.bp
+++ b/Android.bp
@@ -43,7 +43,10 @@
crate_name: "criterion",
srcs: ["src/lib.rs"],
edition: "2018",
- features: ["default"],
+ features: [
+ "cargo_bench_support",
+ "default",
+ ],
rustlibs: [
"libatty",
"libcast",
@@ -71,7 +74,7 @@
// autocfg-1.0.1
// bitflags-1.2.1 "default"
// bstr-0.2.15 "default,lazy_static,regex-automata,serde,serde1,serde1-nostd,std,unicode"
-// byteorder-1.4.2
+// byteorder-1.4.3
// cast-0.2.3 "default,std"
// cfg-if-1.0.0
// clap-2.33.3
@@ -80,40 +83,43 @@
// crossbeam-deque-0.8.0 "crossbeam-epoch,crossbeam-utils,default,std"
// crossbeam-epoch-0.9.3 "alloc,lazy_static,std"
// crossbeam-utils-0.8.3 "default,lazy_static,std"
-// csv-1.1.5
+// csv-1.1.6
// csv-core-0.1.10 "default"
// either-1.6.1
// half-1.7.1
+// itertools-0.10.0 "default,use_alloc,use_std"
// itertools-0.9.0 "default,use_std"
// itoa-0.4.7 "default,std"
// lazy_static-1.4.0
-// libc-0.2.87 "default,std"
-// memchr-2.3.4 "std,use_std"
-// memoffset-0.6.1 "default"
+// libc-0.2.92 "default,std"
+// memchr-2.3.4 "default,std,use_std"
+// memoffset-0.6.3 "default"
// num-traits-0.2.14 "default,std"
// num_cpus-1.13.0
// oorandom-11.1.3
-// plotters-0.2.15 "area_series,line_series,svg"
-// proc-macro2-1.0.24 "default,proc-macro"
+// plotters-0.3.0 "area_series,line_series,plotters-svg,svg_backend"
+// plotters-backend-0.3.0
+// plotters-svg-0.3.0
+// proc-macro2-1.0.26 "default,proc-macro"
// quote-1.0.9 "default,proc-macro"
// rayon-1.5.0
// rayon-core-1.9.0
-// regex-1.4.3 "std"
+// regex-1.4.5 "std"
// regex-automata-0.1.9
-// regex-syntax-0.6.22
+// regex-syntax-0.6.23
// rustc_version-0.2.3
// ryu-1.0.5
// same-file-1.0.6
// scopeguard-1.1.0
// semver-0.9.0 "default"
// semver-parser-0.7.0
-// serde-1.0.123 "default,std"
+// serde-1.0.125 "default,std"
// serde_cbor-0.11.1 "default,std"
-// serde_derive-1.0.123 "default"
+// serde_derive-1.0.125 "default"
// serde_json-1.0.64 "default,std"
-// syn-1.0.61 "clone-impls,default,derive,full,parsing,printing,proc-macro,quote,visit"
+// syn-1.0.68 "clone-impls,default,derive,full,parsing,printing,proc-macro,quote,visit"
// textwrap-0.11.0
// tinytemplate-1.2.1
// unicode-width-0.1.8 "default"
// unicode-xid-0.2.1 "default"
-// walkdir-2.3.1
+// walkdir-2.3.2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4472ef7..a747ed8 100755
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,13 +4,48 @@
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+## [Unreleased]
+
+## [0.3.4]
+### Added
+- Added support for benchmarking async functions
+- Added `with_output_color` for enabling or disabling CLI output coloring programmatically.
+
+### Fixed
+- Criterion.rs will now give a clear error message in case of benchmarks that take zero time.
+- Added some extra code to ensure that every sample has at least one iteration.
+- Added a notice to the `--help` output regarding "unrecognized option" errors.
+- Increased opacity on violin charts.
+- Fixed violin chart X axis not starting at zero in the plotters backend.
+- Criterion.rs will now automatically detect the right output directory.
+
+### Deprecated
+- `Criterion::can_plot` is no longer useful and is deprecated pending deletion in 0.4.0.
+- `Benchmark` and `ParameterizedBenchmark` were already hidden from documentation, but are now
+ formally deprecated pending deletion in 0.4.0. Callers should use `BenchmarkGroup` instead.
+- `Criterion::bench_function_over_inputs`, `Criterion::bench_functions`, and `Criterion::bench` were
+ already hidden from documentation, but are now formally deprecated pending deletion in 0.4.0.
+ Callers should use `BenchmarkGroup` instead.
+- Three new optional features have been added; "html_reports", "csv_output" and
+ "cargo_bench_support". These features currently do nothing except disable a warning message at
+ runtime, but in version 0.4.0 they will be used to enable HTML report generation, CSV file
+ generation, and the ability to run in cargo-bench (as opposed to [cargo-criterion]).
+ "cargo_bench_support" is enabled by default, but "html_reports" and "csv_output"
+ are not. If you use Criterion.rs' HTML reports, it is recommended to switch to [cargo-criterion].
+ If you use CSV output, it is recommended to switch to [cargo-criterion] and use the
+ `--message-format=json` option for machine-readable output instead. A warning message will be
+ printed at the start of benchmark runs which do not have "html_reports" or "cargo_bench_support"
+ enabled, but because CSV output is not widely used it has no warning.
+
+[cargo-criterion]: https://github.com/bheisler/cargo-criterion
+
## [0.3.3] - 2020-06-29
### Added
- Added `CRITERION_HOME` environment variable to set the directory for Criterion to store
its results and charts in.
-- Added support for [cargo-criterion](https://github.com/bheisler/cargo-criterion). The long-term
- goal here is to remove code from Criterion-rs itself to improve compile times, as well as to add
- features to `cargo-criterion` that are difficult to implement in Criterion-rs.
+- Added support for [cargo-criterion]. The long-term goal here is to remove code from Criterion-rs
+ itself to improve compile times, as well as to add features to `cargo-criterion` that are
+ difficult to implement in Criterion-rs.
- Add sampling mode option for benchmarks. This allows the user to change how Criterion.rs chooses
the iteration counts in each sample. By default, nothing will change for most benchmarks, but
very slow benchmarks will now run fewer iterations to fit in the desired number of samples.
@@ -353,7 +388,7 @@
- Initial release on Crates.io.
-[Unreleased]: https://github.com/bheisler/criterion.rs/compare/0.3.3...HEAD
+[Unreleased]: https://github.com/bheisler/criterion.rs/compare/0.3.4...HEAD
[0.1.1]: https://github.com/bheisler/criterion.rs/compare/0.1.0...0.1.1
[0.1.2]: https://github.com/bheisler/criterion.rs/compare/0.1.1...0.1.2
[0.2.0]: https://github.com/bheisler/criterion.rs/compare/0.1.2...0.2.0
@@ -371,4 +406,5 @@
[0.3.0]: https://github.com/bheisler/criterion.rs/compare/0.2.11...0.3.0
[0.3.1]: https://github.com/bheisler/criterion.rs/compare/0.3.0...0.3.1
[0.3.2]: https://github.com/bheisler/criterion.rs/compare/0.3.1...0.3.2
-[0.3.2]: https://github.com/bheisler/criterion.rs/compare/0.3.2...0.3.3
\ No newline at end of file
+[0.3.3]: https://github.com/bheisler/criterion.rs/compare/0.3.2...0.3.3
+[0.3.4]: https://github.com/bheisler/criterion.rs/compare/0.3.3...0.3.4
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3442998..be5026b 100755
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -63,7 +63,7 @@
* Beginner - Suitable for people new to Criterion.rs, or even new to Rust in general
* Intermediate - More challenging, likely involves some non-trivial design decisions and/or knowledge
of Criterion.<span></span>rs' internals
-* Bigger Project - Large and/or complex project such as designing a complex new feature.
+* Bigger Project - Large and/or complex project such as designing a complex new feature
Additionally, there are a few other noteworthy labels:
diff --git a/Cargo.toml b/Cargo.toml
index 4124287..2c8ce7a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@
[package]
edition = "2018"
name = "criterion"
-version = "0.3.3"
+version = "0.3.4"
authors = ["Jorge Aparicio <japaricious@gmail.com>", "Brook Heisler <brookheisler@gmail.com>"]
exclude = ["book/*"]
description = "Statistics-driven micro-benchmarking library"
@@ -23,6 +23,8 @@
categories = ["development-tools::profiling"]
license = "Apache-2.0/MIT"
repository = "https://github.com/bheisler/criterion.rs"
+[package.metadata.docs.rs]
+features = ["async_futures", "async_smol", "async_std", "async_tokio"]
[lib]
bench = false
@@ -30,6 +32,10 @@
[[bench]]
name = "bench_main"
harness = false
+[dependencies.async-std]
+version = "1.9"
+optional = true
+
[dependencies.atty]
version = "0.2"
@@ -46,8 +52,13 @@
[dependencies.csv]
version = "1.1"
+[dependencies.futures]
+version = "0.3"
+optional = true
+default_features = false
+
[dependencies.itertools]
-version = "0.9"
+version = "0.10"
[dependencies.lazy_static]
version = "1.4"
@@ -60,8 +71,8 @@
version = "11.1"
[dependencies.plotters]
-version = "^0.2.12"
-features = ["svg", "area_series", "line_series"]
+version = "^0.3.0"
+features = ["svg_backend", "area_series", "line_series"]
default-features = false
[dependencies.rayon]
@@ -84,26 +95,50 @@
[dependencies.serde_json]
version = "1.0"
+[dependencies.smol]
+version = "1.2"
+optional = true
+default-features = false
+
[dependencies.tinytemplate]
version = "1.1"
+[dependencies.tokio]
+version = "1.0"
+features = ["rt"]
+optional = true
+default-features = false
+
[dependencies.walkdir]
version = "2.3"
[dev-dependencies.approx]
+version = "0.4"
+
+[dev-dependencies.futures]
version = "0.3"
+features = ["executor"]
+default_features = false
[dev-dependencies.quickcheck]
version = "0.9"
default-features = false
[dev-dependencies.rand]
-version = "0.7"
+version = "0.8"
[dev-dependencies.tempfile]
version = "3.1"
[features]
-default = []
+async = ["futures"]
+async_futures = ["futures/executor", "async"]
+async_smol = ["smol", "async"]
+async_std = ["async-std", "async"]
+async_tokio = ["tokio", "async"]
+cargo_bench_support = []
+csv_output = []
+default = ["cargo_bench_support"]
+html_reports = []
real_blackbox = []
[badges.appveyor]
id = "4255ads9ctpupcl2"
diff --git a/Cargo.toml.orig b/Cargo.toml.orig
index 1dc9def..86a8f92 100755
--- a/Cargo.toml.orig
+++ b/Cargo.toml.orig
@@ -1,7 +1,7 @@
[package]
authors = ["Jorge Aparicio <japaricious@gmail.com>", "Brook Heisler <brookheisler@gmail.com>"]
name = "criterion"
-version = "0.3.3"
+version = "0.3.4"
edition = "2018"
description = "Statistics-driven micro-benchmarking library"
@@ -16,7 +16,7 @@
[dependencies]
lazy_static = "1.4"
criterion-plot = { path="plot", version="0.4.3" }
-itertools = "0.9"
+itertools = "0.10"
serde = "1.0"
serde_json = "1.0"
serde_derive = "1.0"
@@ -31,17 +31,22 @@
oorandom = "11.1"
rayon = "1.3"
regex = { version = "1.3", default-features = false, features = ["std"] }
+futures = { version = "0.3", default_features = false, optional = true }
+smol = { version = "1.2", default-features = false, optional = true }
+tokio = { version = "1.0", default-features = false, features = ["rt"], optional = true }
+async-std = { version = "1.9", optional = true }
[dependencies.plotters]
-version = "^0.2.12"
+version = "^0.3.0"
default-features = false
-features = ["svg", "area_series", "line_series"]
+features = ["svg_backend", "area_series", "line_series"]
[dev-dependencies]
tempfile = "3.1"
-approx = "0.3"
+approx = "0.4"
quickcheck = { version = "0.9", default-features = false }
-rand = "0.7"
+rand = "0.8"
+futures = { version = "0.3", default_features = false, features = ["executor"] }
[badges]
travis-ci = { repository = "bheisler/criterion.rs" }
@@ -49,8 +54,33 @@
maintenance = { status = "passively-maintained" }
[features]
+default = ["cargo_bench_support"]
+
+# Enable use of the nightly-only test::black_box function to discourage compiler optimizations.
real_blackbox = []
-default = []
+
+# Enable async/await support
+async = ["futures"]
+
+# These features enable built-in support for running async benchmarks on each different async
+# runtime.
+async_futures = ["futures/executor", "async"]
+async_smol = ["smol", "async"]
+async_tokio = ["tokio", "async"]
+async_std = ["async-std", "async"]
+
+# This feature _currently_ does nothing except disable a warning message, but in 0.4.0 it will be
+# required in order to have Criterion.rs generate its own plots (as opposed to using cargo-criterion)
+html_reports = []
+
+# This feature _currently_ does nothing except disable a warning message, but in 0.4.0 it will be
+# required in order to have Criterion.rs be usable outside of cargo-criterion.
+cargo_bench_support = []
+
+# This feature _currently_ does nothing, but in 0.4.0 it will be
+# required in order to have Criterion.rs generate CSV files. This feature is deprecated in favor of
+# cargo-criterion's --message-format=json option.
+csv_output = []
[workspace]
exclude = ["cargo-criterion"]
@@ -61,3 +91,7 @@
[lib]
bench = false
+
+# Enable all of the async runtimes for the docs.rs output
+[package.metadata.docs.rs]
+features = ["async_futures", "async_smol", "async_std", "async_tokio"]
\ No newline at end of file
diff --git a/METADATA b/METADATA
index 22e5011..2ca3480 100644
--- a/METADATA
+++ b/METADATA
@@ -7,13 +7,13 @@
}
url {
type: ARCHIVE
- value: "https://static.crates.io/crates/criterion/criterion-0.3.3.crate"
+ value: "https://static.crates.io/crates/criterion/criterion-0.3.4.crate"
}
- version: "0.3.3"
+ version: "0.3.4"
license_type: NOTICE
last_upgrade_date {
- year: 2020
- month: 12
- day: 21
+ year: 2021
+ month: 4
+ day: 2
}
}
diff --git a/README.md b/README.md
index e33e84a..bb5bbab 100755
--- a/README.md
+++ b/README.md
@@ -119,7 +119,7 @@
Criterion.<span></span>rs supports the last three stable minor releases of Rust. At time of
writing, this means Rust 1.40 or later. Older versions may work, but are not tested or guaranteed.
-Currently, the oldest version of Rust believed to work is 1.36. Future versions of Criterion.<span></span>rs may
+Currently, the oldest version of Rust believed to work is 1.39. Future versions of Criterion.<span></span>rs may
break support for such old versions, and this will not be considered a breaking change. If you
require Criterion.<span></span>rs to work on old versions of Rust, you will need to stick to a
specific patch version of Criterion.<span></span>rs.
diff --git a/benches/bench_main.rs b/benches/bench_main.rs
index 3d9859b..a153b23 100755
--- a/benches/bench_main.rs
+++ b/benches/bench_main.rs
@@ -13,4 +13,5 @@
benchmarks::measurement_overhead::benches,
benchmarks::custom_measurement::benches,
benchmarks::sampling_mode::benches,
+ benchmarks::async_measurement_overhead::benches,
}
diff --git a/benches/benchmarks/async_measurement_overhead.rs b/benches/benchmarks/async_measurement_overhead.rs
new file mode 100755
index 0000000..0c9605a
--- /dev/null
+++ b/benches/benchmarks/async_measurement_overhead.rs
@@ -0,0 +1,48 @@
+use criterion::{async_executor::FuturesExecutor, criterion_group, BatchSize, Criterion};
+
+fn some_benchmark(c: &mut Criterion) {
+ let mut group = c.benchmark_group("async overhead");
+ group.bench_function("iter", |b| b.to_async(FuturesExecutor).iter(|| async { 1 }));
+ group.bench_function("iter_with_setup", |b| {
+ b.to_async(FuturesExecutor)
+ .iter_with_setup(|| (), |_| async { 1 })
+ });
+ group.bench_function("iter_with_large_setup", |b| {
+ b.to_async(FuturesExecutor)
+ .iter_with_large_setup(|| (), |_| async { 1 })
+ });
+ group.bench_function("iter_with_large_drop", |b| {
+ b.to_async(FuturesExecutor)
+ .iter_with_large_drop(|| async { 1 })
+ });
+ group.bench_function("iter_batched_small_input", |b| {
+ b.to_async(FuturesExecutor)
+ .iter_batched(|| (), |_| async { 1 }, BatchSize::SmallInput)
+ });
+ group.bench_function("iter_batched_large_input", |b| {
+ b.to_async(FuturesExecutor)
+ .iter_batched(|| (), |_| async { 1 }, BatchSize::LargeInput)
+ });
+ group.bench_function("iter_batched_per_iteration", |b| {
+ b.to_async(FuturesExecutor)
+ .iter_batched(|| (), |_| async { 1 }, BatchSize::PerIteration)
+ });
+ group.bench_function("iter_batched_ref_small_input", |b| {
+ b.to_async(FuturesExecutor)
+ .iter_batched_ref(|| (), |_| async { 1 }, BatchSize::SmallInput)
+ });
+ group.bench_function("iter_batched_ref_large_input", |b| {
+ b.to_async(FuturesExecutor)
+ .iter_batched_ref(|| (), |_| async { 1 }, BatchSize::LargeInput)
+ });
+ group.bench_function("iter_batched_ref_per_iteration", |b| {
+ b.to_async(FuturesExecutor).iter_batched_ref(
+ || (),
+ |_| async { 1 },
+ BatchSize::PerIteration,
+ )
+ });
+ group.finish();
+}
+
+criterion_group!(benches, some_benchmark);
diff --git a/benches/benchmarks/compare_functions.rs b/benches/benchmarks/compare_functions.rs
index b3db640..ce44180 100755
--- a/benches/benchmarks/compare_functions.rs
+++ b/benches/benchmarks/compare_functions.rs
@@ -1,3 +1,5 @@
+#![allow(deprecated)]
+
use criterion::{criterion_group, BenchmarkId, Criterion, Fun, ParameterizedBenchmark};
fn fibonacci_slow(n: u64) -> u64 {
diff --git a/benches/benchmarks/iter_with_large_drop.rs b/benches/benchmarks/iter_with_large_drop.rs
index 11de5db..ee9a8e9 100755
--- a/benches/benchmarks/iter_with_large_drop.rs
+++ b/benches/benchmarks/iter_with_large_drop.rs
@@ -1,3 +1,5 @@
+#![allow(deprecated)]
+
use criterion::{criterion_group, Benchmark, Criterion, Throughput};
use std::time::Duration;
diff --git a/benches/benchmarks/iter_with_large_setup.rs b/benches/benchmarks/iter_with_large_setup.rs
index 9cbf51a..217d271 100755
--- a/benches/benchmarks/iter_with_large_setup.rs
+++ b/benches/benchmarks/iter_with_large_setup.rs
@@ -1,3 +1,5 @@
+#![allow(deprecated)]
+
use criterion::{criterion_group, Benchmark, Criterion, Throughput};
use std::time::Duration;
diff --git a/benches/benchmarks/mod.rs b/benches/benchmarks/mod.rs
index 5ff243f..ef85910 100755
--- a/benches/benchmarks/mod.rs
+++ b/benches/benchmarks/mod.rs
@@ -8,3 +8,14 @@
pub mod sampling_mode;
pub mod special_characters;
pub mod with_inputs;
+
+#[cfg(feature = "async_futures")]
+pub mod async_measurement_overhead;
+
+#[cfg(not(feature = "async_futures"))]
+pub mod async_measurement_overhead {
+ use criterion::{criterion_group, Criterion};
+ fn some_benchmark(_c: &mut Criterion) {}
+
+ criterion_group!(benches, some_benchmark);
+}
diff --git a/ci/install.sh b/ci/install.sh
index aee01b3..f82a80d 100755
--- a/ci/install.sh
+++ b/ci/install.sh
@@ -12,6 +12,9 @@
if [ "$DOCS" = "yes" ]; then
cargo install mdbook --no-default-features
cargo install mdbook-linkcheck
+ sudo apt-get update
+ sudo apt-get install python-pip
+ sudo pip install python-dateutil
fi
if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$GNUPLOT" = "yes" ]; then
diff --git a/ci/script.sh b/ci/script.sh
index 0bf34e0..b9f6886 100755
--- a/ci/script.sh
+++ b/ci/script.sh
@@ -1,10 +1,14 @@
set -ex
+export CARGO_INCREMENTAL=0
+
+FEATURES="async_smol async_tokio async_std async_futures"
+
if [ "$CLIPPY" = "yes" ]; then
cargo clippy --all -- -D warnings
elif [ "$DOCS" = "yes" ]; then
cargo clean
- cargo doc --all --no-deps
+ cargo doc --features "$FEATURES" --all --no-deps
cd book
mdbook build
cd ..
@@ -18,10 +22,10 @@
else
export RUSTFLAGS="-D warnings"
- cargo build $BUILD_ARGS --release
+ cargo build --features "$FEATURES" $BUILD_ARGS
- cargo test --all --release
- cargo test --benches
+ cargo test --features "$FEATURES" --all
+ cargo test --features "$FEATURES" --benches
cd bencher_compat
export CARGO_TARGET_DIR="../target"
diff --git a/patches/cargo_pkg_version.patch b/patches/cargo_pkg_version.patch
index a664455..dc99390 100644
--- a/patches/cargo_pkg_version.patch
+++ b/patches/cargo_pkg_version.patch
@@ -1,5 +1,5 @@
---- src/connection.rs 2020-12-21 14:55:17.022985801 +0100
-+++ src/connection.rs 2020-12-21 14:55:35.150841529 +0100
+--- a/src/connection.rs 2020-12-21 14:55:17.022985801 +0100
++++ b/src/connection.rs 2020-12-21 14:55:35.150841529 +0100
@@ -84,9 +84,9 @@
hello_buf[0..BENCHMARK_MAGIC_NUMBER.len()]
.copy_from_slice(BENCHMARK_MAGIC_NUMBER.as_bytes());
diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs
index caa948d..1c072e1 100755
--- a/src/analysis/mod.rs
+++ b/src/analysis/mod.rs
@@ -13,7 +13,7 @@
};
use crate::fs;
use crate::measurement::Measurement;
-use crate::report::{BenchmarkId, ReportContext};
+use crate::report::{BenchmarkId, Report, ReportContext};
use crate::routine::Routine;
use crate::{Baseline, Criterion, SavedSample, Throughput};
@@ -105,11 +105,22 @@
conn.serve_value_formatter(criterion.measurement.formatter())
.unwrap();
+ return;
}
}
criterion.report.analysis(id, report_context);
+ if times.iter().any(|&f| f == 0.0) {
+ error!(
+ "At least one measurement of benchmark {} took zero time per \
+ iteration. This should not be possible. If using iter_custom, please verify \
+ that your routine is correctly measured.",
+ id.as_title()
+ );
+ return;
+ }
+
let avg_times = iters
.iter()
.zip(times.iter())
diff --git a/src/async_executor.rs b/src/async_executor.rs
new file mode 100755
index 0000000..127af27
--- /dev/null
+++ b/src/async_executor.rs
@@ -0,0 +1,66 @@
+//! This module defines a trait that can be used to plug in different Futures executors into
+//! Criterion.rs' async benchmarking support.
+//!
+//! Implementations are provided for:
+//! * Tokio (implemented directly for tokio::Runtime)
+//! * Async-std
+//! * Smol
+//! * The Futures crate
+//!
+//! Please note that async benchmarks will have a small amount of measurement overhead relative
+//! to synchronous benchmarks. It is recommended to use synchronous benchmarks where possible, to
+//! improve measurement accuracy.
+
+use std::future::Future;
+
+/// Plugin trait used to allow benchmarking on multiple different async runtimes.
+///
+/// Smol, Tokio and Async-std are supported out of the box, as is the current-thread runner from the
+/// Futures crate; it is recommended to use whichever runtime you use in production.
+pub trait AsyncExecutor {
+ /// Spawn the given future onto this runtime and block until it's complete, returning the result.
+ fn block_on<T>(&self, future: impl Future<Output = T>) -> T;
+}
+
+/// Runs futures on the 'futures' crate's built-in current-thread executor
+#[cfg(feature = "async_futures")]
+pub struct FuturesExecutor;
+#[cfg(feature = "async_futures")]
+impl AsyncExecutor for FuturesExecutor {
+ fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+ futures::executor::block_on(future)
+ }
+}
+
+/// Runs futures on the 'soml' crate's global executor
+#[cfg(feature = "async_smol")]
+pub struct SmolExecutor;
+#[cfg(feature = "async_smol")]
+impl AsyncExecutor for SmolExecutor {
+ fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+ smol::block_on(future)
+ }
+}
+
+#[cfg(feature = "async_tokio")]
+impl AsyncExecutor for tokio::runtime::Runtime {
+ fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+ self.block_on(future)
+ }
+}
+#[cfg(feature = "async_tokio")]
+impl AsyncExecutor for &tokio::runtime::Runtime {
+ fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+ (*self).block_on(future)
+ }
+}
+
+/// Runs futures on the 'async-std' crate's global executor
+#[cfg(feature = "async_std")]
+pub struct AsyncStdExecutor;
+#[cfg(feature = "async_std")]
+impl AsyncExecutor for AsyncStdExecutor {
+ fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+ async_std::task::block_on(future)
+ }
+}
diff --git a/src/bencher.rs b/src/bencher.rs
new file mode 100755
index 0000000..fa9ad3d
--- /dev/null
+++ b/src/bencher.rs
@@ -0,0 +1,772 @@
+use std::iter::IntoIterator;
+use std::time::Duration;
+use std::time::Instant;
+
+use crate::black_box;
+use crate::measurement::{Measurement, WallTime};
+use crate::BatchSize;
+
+#[cfg(feature = "async")]
+use std::future::Future;
+
+#[cfg(feature = "async")]
+use crate::async_executor::AsyncExecutor;
+
+// ================================== MAINTENANCE NOTE =============================================
+// Any changes made to either Bencher or AsyncBencher will have to be replicated to the other!
+// ================================== MAINTENANCE NOTE =============================================
+
+/// Timer struct used to iterate a benchmarked function and measure the runtime.
+///
+/// This struct provides different timing loops as methods. Each timing loop provides a different
+/// way to time a routine and each has advantages and disadvantages.
+///
+/// * If you want to do the iteration and measurement yourself (eg. passing the iteration count
+/// to a separate process), use `iter_custom`.
+/// * If your routine requires no per-iteration setup and returns a value with an expensive `drop`
+/// method, use `iter_with_large_drop`.
+/// * If your routine requires some per-iteration setup that shouldn't be timed, use `iter_batched`
+/// or `iter_batched_ref`. See [`BatchSize`](enum.BatchSize.html) for a discussion of batch sizes.
+/// If the setup value implements `Drop` and you don't want to include the `drop` time in the
+/// measurement, use `iter_batched_ref`, otherwise use `iter_batched`. These methods are also
+/// suitable for benchmarking routines which return a value with an expensive `drop` method,
+/// but are more complex than `iter_with_large_drop`.
+/// * Otherwise, use `iter`.
+pub struct Bencher<'a, M: Measurement = WallTime> {
+ pub(crate) iterated: bool, // Have we iterated this benchmark?
+ pub(crate) iters: u64, // Number of times to iterate this benchmark
+ pub(crate) value: M::Value, // The measured value
+ pub(crate) measurement: &'a M, // Reference to the measurement object
+ pub(crate) elapsed_time: Duration, // How much time did it take to perform the iteration? Used for the warmup period.
+}
+impl<'a, M: Measurement> Bencher<'a, M> {
+ /// Times a `routine` by executing it many times and timing the total elapsed time.
+ ///
+ /// Prefer this timing loop when `routine` returns a value that doesn't have a destructor.
+ ///
+ /// # Timing model
+ ///
+ /// Note that the `Bencher` also times the time required to destroy the output of `routine()`.
+ /// Therefore prefer this timing loop when the runtime of `mem::drop(O)` is negligible compared
+ /// to the runtime of the `routine`.
+ ///
+ /// ```text
+ /// elapsed = Instant::now + iters * (routine + mem::drop(O) + Range::next)
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ ///
+ /// // The function to benchmark
+ /// fn foo() {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("iter", move |b| {
+ /// b.iter(|| foo())
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter<O, R>(&mut self, mut routine: R)
+ where
+ R: FnMut() -> O,
+ {
+ self.iterated = true;
+ let time_start = Instant::now();
+ let start = self.measurement.start();
+ for _ in 0..self.iters {
+ black_box(routine());
+ }
+ self.value = self.measurement.end(start);
+ self.elapsed_time = time_start.elapsed();
+ }
+
+ /// Times a `routine` by executing it many times and relying on `routine` to measure its own execution time.
+ ///
+ /// Prefer this timing loop in cases where `routine` has to do its own measurements to
+ /// get accurate timing information (for example in multi-threaded scenarios where you spawn
+ /// and coordinate with multiple threads).
+ ///
+ /// # Timing model
+ /// Custom, the timing model is whatever is returned as the Duration from `routine`.
+ ///
+ /// # Example
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ /// use criterion::*;
+ /// use criterion::black_box;
+ /// use std::time::Instant;
+ ///
+ /// fn foo() {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("iter", move |b| {
+ /// b.iter_custom(|iters| {
+ /// let start = Instant::now();
+ /// for _i in 0..iters {
+ /// black_box(foo());
+ /// }
+ /// start.elapsed()
+ /// })
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_custom<R>(&mut self, mut routine: R)
+ where
+ R: FnMut(u64) -> M::Value,
+ {
+ self.iterated = true;
+ let time_start = Instant::now();
+ self.value = routine(self.iters);
+ self.elapsed_time = time_start.elapsed();
+ }
+
+ #[doc(hidden)]
+ pub fn iter_with_setup<I, O, S, R>(&mut self, setup: S, routine: R)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> O,
+ {
+ self.iter_batched(setup, routine, BatchSize::PerIteration);
+ }
+
+ /// Times a `routine` by collecting its output on each iteration. This avoids timing the
+ /// destructor of the value returned by `routine`.
+ ///
+ /// WARNING: This requires `O(iters * mem::size_of::<O>())` of memory, and `iters` is not under the
+ /// control of the caller. If this causes out-of-memory errors, use `iter_batched` instead.
+ ///
+ /// # Timing model
+ ///
+ /// ``` text
+ /// elapsed = Instant::now + iters * (routine) + Iterator::collect::<Vec<_>>
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ ///
+ /// fn create_vector() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("with_drop", move |b| {
+ /// // This will avoid timing the Vec::drop.
+ /// b.iter_with_large_drop(|| create_vector())
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ pub fn iter_with_large_drop<O, R>(&mut self, mut routine: R)
+ where
+ R: FnMut() -> O,
+ {
+ self.iter_batched(|| (), |_| routine(), BatchSize::SmallInput);
+ }
+
+ #[doc(hidden)]
+ pub fn iter_with_large_setup<I, O, S, R>(&mut self, setup: S, routine: R)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> O,
+ {
+ self.iter_batched(setup, routine, BatchSize::NumBatches(1));
+ }
+
+ /// Times a `routine` that requires some input by generating a batch of input, then timing the
+ /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for
+ /// details on choosing the batch size. Use this when the routine must consume its input.
+ ///
+ /// For example, use this loop to benchmark sorting algorithms, because they require unsorted
+ /// data on each iteration.
+ ///
+ /// # Timing model
+ ///
+ /// ```text
+ /// elapsed = (Instant::now * num_batches) + (iters * (routine + O::drop)) + Vec::extend
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ ///
+ /// fn create_scrambled_data() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// // The sorting algorithm to test
+ /// fn sort(data: &mut [u64]) {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// let data = create_scrambled_data();
+ ///
+ /// c.bench_function("with_setup", move |b| {
+ /// // This will avoid timing the to_vec call.
+ /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput)
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_batched<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> O,
+ {
+ self.iterated = true;
+ let batch_size = size.iters_per_batch(self.iters);
+ assert!(batch_size != 0, "Batch size must not be zero.");
+ let time_start = Instant::now();
+ self.value = self.measurement.zero();
+
+ if batch_size == 1 {
+ for _ in 0..self.iters {
+ let input = black_box(setup());
+
+ let start = self.measurement.start();
+ let output = routine(input);
+ let end = self.measurement.end(start);
+ self.value = self.measurement.add(&self.value, &end);
+
+ drop(black_box(output));
+ }
+ } else {
+ let mut iteration_counter = 0;
+
+ while iteration_counter < self.iters {
+ let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter);
+
+ let inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>());
+ let mut outputs = Vec::with_capacity(batch_size as usize);
+
+ let start = self.measurement.start();
+ outputs.extend(inputs.into_iter().map(&mut routine));
+ let end = self.measurement.end(start);
+ self.value = self.measurement.add(&self.value, &end);
+
+ black_box(outputs);
+
+ iteration_counter += batch_size;
+ }
+ }
+
+ self.elapsed_time = time_start.elapsed();
+ }
+
+ /// Times a `routine` that requires some input by generating a batch of input, then timing the
+ /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for
+ /// details on choosing the batch size. Use this when the routine should accept the input by
+ /// mutable reference.
+ ///
+ /// For example, use this loop to benchmark sorting algorithms, because they require unsorted
+ /// data on each iteration.
+ ///
+ /// # Timing model
+ ///
+ /// ```text
+ /// elapsed = (Instant::now * num_batches) + (iters * routine) + Vec::extend
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ ///
+ /// fn create_scrambled_data() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// // The sorting algorithm to test
+ /// fn sort(data: &mut [u64]) {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// let data = create_scrambled_data();
+ ///
+ /// c.bench_function("with_setup", move |b| {
+ /// // This will avoid timing the to_vec call.
+ /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput)
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_batched_ref<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize)
+ where
+ S: FnMut() -> I,
+ R: FnMut(&mut I) -> O,
+ {
+ self.iterated = true;
+ let batch_size = size.iters_per_batch(self.iters);
+ assert!(batch_size != 0, "Batch size must not be zero.");
+ let time_start = Instant::now();
+ self.value = self.measurement.zero();
+
+ if batch_size == 1 {
+ for _ in 0..self.iters {
+ let mut input = black_box(setup());
+
+ let start = self.measurement.start();
+ let output = routine(&mut input);
+ let end = self.measurement.end(start);
+ self.value = self.measurement.add(&self.value, &end);
+
+ drop(black_box(output));
+ drop(black_box(input));
+ }
+ } else {
+ let mut iteration_counter = 0;
+
+ while iteration_counter < self.iters {
+ let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter);
+
+ let mut inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>());
+ let mut outputs = Vec::with_capacity(batch_size as usize);
+
+ let start = self.measurement.start();
+ outputs.extend(inputs.iter_mut().map(&mut routine));
+ let end = self.measurement.end(start);
+ self.value = self.measurement.add(&self.value, &end);
+
+ black_box(outputs);
+
+ iteration_counter += batch_size;
+ }
+ }
+ self.elapsed_time = time_start.elapsed();
+ }
+
+ // Benchmarks must actually call one of the iter methods. This causes benchmarks to fail loudly
+ // if they don't.
+ pub(crate) fn assert_iterated(&mut self) {
+ if !self.iterated {
+ panic!("Benchmark function must call Bencher::iter or related method.");
+ }
+ self.iterated = false;
+ }
+
+ /// Convert this bencher into an AsyncBencher, which enables async/await support.
+ #[cfg(feature = "async")]
+ pub fn to_async<'b, A: AsyncExecutor>(&'b mut self, runner: A) -> AsyncBencher<'a, 'b, A, M> {
+ AsyncBencher { b: self, runner }
+ }
+}
+
+/// Async/await variant of the Bencher struct.
+#[cfg(feature = "async")]
+pub struct AsyncBencher<'a, 'b, A: AsyncExecutor, M: Measurement = WallTime> {
+ b: &'b mut Bencher<'a, M>,
+ runner: A,
+}
+#[cfg(feature = "async")]
+impl<'a, 'b, A: AsyncExecutor, M: Measurement> AsyncBencher<'a, 'b, A, M> {
+ /// Times a `routine` by executing it many times and timing the total elapsed time.
+ ///
+ /// Prefer this timing loop when `routine` returns a value that doesn't have a destructor.
+ ///
+ /// # Timing model
+ ///
+ /// Note that the `AsyncBencher` also times the time required to destroy the output of `routine()`.
+ /// Therefore prefer this timing loop when the runtime of `mem::drop(O)` is negligible compared
+ /// to the runtime of the `routine`.
+ ///
+ /// ```text
+ /// elapsed = Instant::now + iters * (routine + mem::drop(O) + Range::next)
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ /// use criterion::async_executor::FuturesExecutor;
+ ///
+ /// // The function to benchmark
+ /// async fn foo() {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("iter", move |b| {
+ /// b.to_async(FuturesExecutor).iter(|| async { foo().await } )
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter<O, R, F>(&mut self, mut routine: R)
+ where
+ R: FnMut() -> F,
+ F: Future<Output = O>,
+ {
+ let AsyncBencher { b, runner } = self;
+ runner.block_on(async {
+ b.iterated = true;
+ let time_start = Instant::now();
+ let start = b.measurement.start();
+ for _ in 0..b.iters {
+ black_box(routine().await);
+ }
+ b.value = b.measurement.end(start);
+ b.elapsed_time = time_start.elapsed();
+ });
+ }
+
+ /// Times a `routine` by executing it many times and relying on `routine` to measure its own execution time.
+ ///
+ /// Prefer this timing loop in cases where `routine` has to do its own measurements to
+ /// get accurate timing information (for example in multi-threaded scenarios where you spawn
+ /// and coordinate with multiple threads).
+ ///
+ /// # Timing model
+ /// Custom, the timing model is whatever is returned as the Duration from `routine`.
+ ///
+ /// # Example
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ /// use criterion::*;
+ /// use criterion::black_box;
+ /// use criterion::async_executor::FuturesExecutor;
+ /// use std::time::Instant;
+ ///
+ /// async fn foo() {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("iter", move |b| {
+ /// b.to_async(FuturesExecutor).iter_custom(|iters| {
+ /// async move {
+ /// let start = Instant::now();
+ /// for _i in 0..iters {
+ /// black_box(foo().await);
+ /// }
+ /// start.elapsed()
+ /// }
+ /// })
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_custom<R, F>(&mut self, mut routine: R)
+ where
+ R: FnMut(u64) -> F,
+ F: Future<Output = M::Value>,
+ {
+ let AsyncBencher { b, runner } = self;
+ runner.block_on(async {
+ b.iterated = true;
+ let time_start = Instant::now();
+ b.value = routine(b.iters).await;
+ b.elapsed_time = time_start.elapsed();
+ })
+ }
+
+ #[doc(hidden)]
+ pub fn iter_with_setup<I, O, S, R, F>(&mut self, setup: S, routine: R)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> F,
+ F: Future<Output = O>,
+ {
+ self.iter_batched(setup, routine, BatchSize::PerIteration);
+ }
+
+ /// Times a `routine` by collecting its output on each iteration. This avoids timing the
+ /// destructor of the value returned by `routine`.
+ ///
+ /// WARNING: This requires `O(iters * mem::size_of::<O>())` of memory, and `iters` is not under the
+ /// control of the caller. If this causes out-of-memory errors, use `iter_batched` instead.
+ ///
+ /// # Timing model
+ ///
+ /// ``` text
+ /// elapsed = Instant::now + iters * (routine) + Iterator::collect::<Vec<_>>
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ /// use criterion::async_executor::FuturesExecutor;
+ ///
+ /// async fn create_vector() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// c.bench_function("with_drop", move |b| {
+ /// // This will avoid timing the Vec::drop.
+ /// b.to_async(FuturesExecutor).iter_with_large_drop(|| async { create_vector().await })
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ pub fn iter_with_large_drop<O, R, F>(&mut self, mut routine: R)
+ where
+ R: FnMut() -> F,
+ F: Future<Output = O>,
+ {
+ self.iter_batched(|| (), |_| routine(), BatchSize::SmallInput);
+ }
+
+ #[doc(hidden)]
+ pub fn iter_with_large_setup<I, O, S, R, F>(&mut self, setup: S, routine: R)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> F,
+ F: Future<Output = O>,
+ {
+ self.iter_batched(setup, routine, BatchSize::NumBatches(1));
+ }
+
+ /// Times a `routine` that requires some input by generating a batch of input, then timing the
+ /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for
+ /// details on choosing the batch size. Use this when the routine must consume its input.
+ ///
+ /// For example, use this loop to benchmark sorting algorithms, because they require unsorted
+ /// data on each iteration.
+ ///
+ /// # Timing model
+ ///
+ /// ```text
+ /// elapsed = (Instant::now * num_batches) + (iters * (routine + O::drop)) + Vec::extend
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ /// use criterion::async_executor::FuturesExecutor;
+ ///
+ /// fn create_scrambled_data() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// // The sorting algorithm to test
+ /// async fn sort(data: &mut [u64]) {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// let data = create_scrambled_data();
+ ///
+ /// c.bench_function("with_setup", move |b| {
+ /// // This will avoid timing the to_vec call.
+ /// b.iter_batched(|| data.clone(), |mut data| async move { sort(&mut data).await }, BatchSize::SmallInput)
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_batched<I, O, S, R, F>(&mut self, mut setup: S, mut routine: R, size: BatchSize)
+ where
+ S: FnMut() -> I,
+ R: FnMut(I) -> F,
+ F: Future<Output = O>,
+ {
+ let AsyncBencher { b, runner } = self;
+ runner.block_on(async {
+ b.iterated = true;
+ let batch_size = size.iters_per_batch(b.iters);
+ assert!(batch_size != 0, "Batch size must not be zero.");
+ let time_start = Instant::now();
+ b.value = b.measurement.zero();
+
+ if batch_size == 1 {
+ for _ in 0..b.iters {
+ let input = black_box(setup());
+
+ let start = b.measurement.start();
+ let output = routine(input).await;
+ let end = b.measurement.end(start);
+ b.value = b.measurement.add(&b.value, &end);
+
+ drop(black_box(output));
+ }
+ } else {
+ let mut iteration_counter = 0;
+
+ while iteration_counter < b.iters {
+ let batch_size = ::std::cmp::min(batch_size, b.iters - iteration_counter);
+
+ let inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>());
+ let mut outputs = Vec::with_capacity(batch_size as usize);
+
+ let start = b.measurement.start();
+ // Can't use .extend here like the sync version does
+ for input in inputs {
+ outputs.push(routine(input).await);
+ }
+ let end = b.measurement.end(start);
+ b.value = b.measurement.add(&b.value, &end);
+
+ black_box(outputs);
+
+ iteration_counter += batch_size;
+ }
+ }
+
+ b.elapsed_time = time_start.elapsed();
+ })
+ }
+
+ /// Times a `routine` that requires some input by generating a batch of input, then timing the
+ /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for
+ /// details on choosing the batch size. Use this when the routine should accept the input by
+ /// mutable reference.
+ ///
+ /// For example, use this loop to benchmark sorting algorithms, because they require unsorted
+ /// data on each iteration.
+ ///
+ /// # Timing model
+ ///
+ /// ```text
+ /// elapsed = (Instant::now * num_batches) + (iters * routine) + Vec::extend
+ /// ```
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// #[macro_use] extern crate criterion;
+ ///
+ /// use criterion::*;
+ /// use criterion::async_executor::FuturesExecutor;
+ ///
+ /// fn create_scrambled_data() -> Vec<u64> {
+ /// # vec![]
+ /// // ...
+ /// }
+ ///
+ /// // The sorting algorithm to test
+ /// async fn sort(data: &mut [u64]) {
+ /// // ...
+ /// }
+ ///
+ /// fn bench(c: &mut Criterion) {
+ /// let data = create_scrambled_data();
+ ///
+ /// c.bench_function("with_setup", move |b| {
+ /// // This will avoid timing the to_vec call.
+ /// b.iter_batched(|| data.clone(), |mut data| async move { sort(&mut data).await }, BatchSize::SmallInput)
+ /// });
+ /// }
+ ///
+ /// criterion_group!(benches, bench);
+ /// criterion_main!(benches);
+ /// ```
+ ///
+ #[inline(never)]
+ pub fn iter_batched_ref<I, O, S, R, F>(&mut self, mut setup: S, mut routine: R, size: BatchSize)
+ where
+ S: FnMut() -> I,
+ R: FnMut(&mut I) -> F,
+ F: Future<Output = O>,
+ {
+ let AsyncBencher { b, runner } = self;
+ runner.block_on(async {
+ b.iterated = true;
+ let batch_size = size.iters_per_batch(b.iters);
+ assert!(batch_size != 0, "Batch size must not be zero.");
+ let time_start = Instant::now();
+ b.value = b.measurement.zero();
+
+ if batch_size == 1 {
+ for _ in 0..b.iters {
+ let mut input = black_box(setup());
+
+ let start = b.measurement.start();
+ let output = routine(&mut input).await;
+ let end = b.measurement.end(start);
+ b.value = b.measurement.add(&b.value, &end);
+
+ drop(black_box(output));
+ drop(black_box(input));
+ }
+ } else {
+ let mut iteration_counter = 0;
+
+ while iteration_counter < b.iters {
+ let batch_size = ::std::cmp::min(batch_size, b.iters - iteration_counter);
+
+ let inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>());
+ let mut outputs = Vec::with_capacity(batch_size as usize);
+
+ let start = b.measurement.start();
+ // Can't use .extend here like the sync version does
+ for mut input in inputs {
+ outputs.push(routine(&mut input).await);
+ }
+ let end = b.measurement.end(start);
+ b.value = b.measurement.add(&b.value, &end);
+
+ black_box(outputs);
+
+ iteration_counter += batch_size;
+ }
+ }
+ b.elapsed_time = time_start.elapsed();
+ });
+ }
+}
diff --git a/src/benchmark.rs b/src/benchmark.rs
index a99750d..bc7f610 100755
--- a/src/benchmark.rs
+++ b/src/benchmark.rs
@@ -1,7 +1,9 @@
+#![allow(deprecated)]
+
use crate::analysis;
use crate::connection::OutgoingMessage;
use crate::measurement::{Measurement, WallTime};
-use crate::report::{BenchmarkId, ReportContext};
+use crate::report::{BenchmarkId, Report, ReportContext};
use crate::routine::{Function, Routine};
use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput};
use std::cell::RefCell;
@@ -78,6 +80,7 @@
/// Structure representing a benchmark (or group of benchmarks)
/// which take one parameter.
#[doc(hidden)]
+#[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")]
pub struct ParameterizedBenchmark<T: Debug, M: Measurement = WallTime> {
config: PartialBenchmarkConfig,
values: Vec<T>,
@@ -88,6 +91,7 @@
/// Structure representing a benchmark (or group of benchmarks)
/// which takes no parameters.
#[doc(hidden)]
+#[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")]
pub struct Benchmark<M: Measurement = WallTime> {
config: PartialBenchmarkConfig,
routines: Vec<NamedRoutine<(), M>>,
@@ -134,7 +138,7 @@
}
/// Changes the target measurement time for this benchmark. Criterion will attempt
- /// to spent approximately this amount of time measuring the benchmark.
+ /// to spend approximately this amount of time measuring the benchmark.
/// With a longer time, the measurement will become more resilient to transitory peak loads
/// caused by external programs.
///
@@ -216,7 +220,7 @@
/// noise.
///
/// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
- /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to
+ /// the statistical robustness against noise, but it also weakens Criterion.rs' ability to
/// detect small but real changes in the performance. By setting the significance level
/// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
/// report more spurious differences.
diff --git a/src/benchmark_group.rs b/src/benchmark_group.rs
index c59cf56..ecb8258 100755
--- a/src/benchmark_group.rs
+++ b/src/benchmark_group.rs
@@ -3,6 +3,7 @@
use crate::connection::OutgoingMessage;
use crate::measurement::Measurement;
use crate::report::BenchmarkId as InternalBenchmarkId;
+use crate::report::Report;
use crate::report::ReportContext;
use crate::routine::{Function, Routine};
use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput};
@@ -198,7 +199,7 @@
/// noise.
///
/// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
- /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to
+ /// the statistical robustness against noise, but it also weakens Criterion.rs' ability to
/// detect small but real changes in the performance. By setting the significance level
/// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
/// report more spurious differences.
diff --git a/src/html/benchmark_report.html.tt b/src/html/benchmark_report.html.tt
index 5b0679e..babd003 100755
--- a/src/html/benchmark_report.html.tt
+++ b/src/html/benchmark_report.html.tt
@@ -187,7 +187,7 @@
<section class="explanation">
<h4>Understanding this report:</h4>
<p>The plot on the left displays the average time per iteration for this benchmark. The shaded region
- shows the estimated probabilty of an iteration taking a certain amount of time, while the line
+ shows the estimated probability of an iteration taking a certain amount of time, while the line
shows the mean. Click on the plot for a larger view showing the outliers.</p>
{{- if slope }}
<p>The plot on the right shows the linear regression calculated from the measurements. Each point
diff --git a/src/lib.rs b/src/lib.rs
index 0fd273c..7cc4070 100755
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -11,7 +11,7 @@
//!
//! ## Features:
//! * Collects detailed statistics, providing strong confidence that changes
-//! to performance are real, not measurement noise
+//! to performance are real, not measurement noise.
//! * Produces detailed charts, providing thorough understanding of your code's
//! performance behavior.
@@ -24,6 +24,8 @@
clippy::just_underscores_and_digits, // Used in the stats code
clippy::transmute_ptr_to_ptr, // Used in the stats code
clippy::option_as_ref_deref, // Remove when MSRV bumped above 1.40
+ clippy::manual_non_exhaustive, // Remove when MSRV bumped above 1.40
+ clippy::match_like_matches_macro, // Remove when MSRV bumped above 1.42
)
)]
@@ -54,6 +56,8 @@
mod benchmark;
#[macro_use]
mod benchmark_group;
+pub mod async_executor;
+mod bencher;
mod connection;
mod csv_report;
mod error;
@@ -73,14 +77,15 @@
use std::cell::RefCell;
use std::collections::HashSet;
use std::default::Default;
+use std::env;
use std::fmt;
use std::iter::IntoIterator;
use std::marker::PhantomData;
use std::net::TcpStream;
use std::path::{Path, PathBuf};
+use std::process::Command;
use std::sync::{Mutex, MutexGuard};
use std::time::Duration;
-use std::time::Instant;
use criterion_plot::{Version, VersionError};
@@ -96,6 +101,10 @@
use crate::report::{BencherReport, CliReport, Report, ReportContext, Reports};
use crate::routine::Function;
+#[cfg(feature = "async")]
+pub use crate::bencher::AsyncBencher;
+pub use crate::bencher::Bencher;
+#[allow(deprecated)]
pub use crate::benchmark::{Benchmark, BenchmarkDefinition, ParameterizedBenchmark};
pub use crate::benchmark_group::{BenchmarkGroup, BenchmarkId};
@@ -127,6 +136,20 @@
Err(_) => None,
}
};
+ static ref DEFAULT_OUTPUT_DIRECTORY: PathBuf = {
+ // Set criterion home to (in descending order of preference):
+ // - $CRITERION_HOME (cargo-criterion sets this, but other users could as well)
+ // - $CARGO_TARGET_DIR/criterion
+ // - the cargo target dir from `cargo metadata`
+ // - ./target/criterion
+ if let Some(value) = env::var_os("CRITERION_HOME") {
+ PathBuf::from(value)
+ } else if let Some(path) = cargo_target_directory() {
+ path.join("criterion")
+ } else {
+ PathBuf::from("target/criterion")
+ }
+ };
}
fn debug_enabled() -> bool {
@@ -245,7 +268,7 @@
/// `NumIterations` fixes the batch size to a constant number, specified by the user. This
/// allows the user to choose their own tradeoff between overhead and memory usage, but care must
- /// be taken in tuning the batch size. In general, the measurement overhead of NumIterations
+ /// be taken in tuning the batch size. In general, the measurement overhead of `NumIterations`
/// will be larger than that of `NumBatches`. Most benchmarks should use `SmallInput` or
/// `LargeInput` instead.
NumIterations(u64),
@@ -272,377 +295,6 @@
}
}
-/// Timer struct used to iterate a benchmarked function and measure the runtime.
-///
-/// This struct provides different timing loops as methods. Each timing loop provides a different
-/// way to time a routine and each has advantages and disadvantages.
-///
-/// * If you want to do the iteration and measurement yourself (eg. passing the iteration count
-/// to a separate process), use `iter_custom`.
-/// * If your routine requires no per-iteration setup and returns a value with an expensive `drop`
-/// method, use `iter_with_large_drop`.
-/// * If your routine requires some per-iteration setup that shouldn't be timed, use `iter_batched`
-/// or `iter_batched_ref`. See [`BatchSize`](enum.BatchSize.html) for a discussion of batch sizes.
-/// If the setup value implements `Drop` and you don't want to include the `drop` time in the
-/// measurement, use `iter_batched_ref`, otherwise use `iter_batched`. These methods are also
-/// suitable for benchmarking routines which return a value with an expensive `drop` method,
-/// but are more complex than `iter_with_large_drop`.
-/// * Otherwise, use `iter`.
-pub struct Bencher<'a, M: Measurement = WallTime> {
- iterated: bool, // have we iterated this benchmark?
- iters: u64, // Number of times to iterate this benchmark
- value: M::Value, // The measured value
- measurement: &'a M, // Reference to the measurement object
- elapsed_time: Duration, // How much time did it take to perform the iteration? Used for the warmup period.
-}
-impl<'a, M: Measurement> Bencher<'a, M> {
- /// Times a `routine` by executing it many times and timing the total elapsed time.
- ///
- /// Prefer this timing loop when `routine` returns a value that doesn't have a destructor.
- ///
- /// # Timing model
- ///
- /// Note that the `Bencher` also times the time required to destroy the output of `routine()`.
- /// Therefore prefer this timing loop when the runtime of `mem::drop(O)` is negligible compared
- /// to the runtime of the `routine`.
- ///
- /// ```text
- /// elapsed = Instant::now + iters * (routine + mem::drop(O) + Range::next)
- /// ```
- ///
- /// # Example
- ///
- /// ```rust
- /// #[macro_use] extern crate criterion;
- ///
- /// use criterion::*;
- ///
- /// // The function to benchmark
- /// fn foo() {
- /// // ...
- /// }
- ///
- /// fn bench(c: &mut Criterion) {
- /// c.bench_function("iter", move |b| {
- /// b.iter(|| foo())
- /// });
- /// }
- ///
- /// criterion_group!(benches, bench);
- /// criterion_main!(benches);
- /// ```
- ///
- #[inline(never)]
- pub fn iter<O, R>(&mut self, mut routine: R)
- where
- R: FnMut() -> O,
- {
- self.iterated = true;
- let time_start = Instant::now();
- let start = self.measurement.start();
- for _ in 0..self.iters {
- black_box(routine());
- }
- self.value = self.measurement.end(start);
- self.elapsed_time = time_start.elapsed();
- }
-
- /// Times a `routine` by executing it many times and relying on `routine` to measure its own execution time.
- ///
- /// Prefer this timing loop in cases where `routine` has to do its own measurements to
- /// get accurate timing information (for example in multi-threaded scenarios where you spawn
- /// and coordinate with multiple threads).
- ///
- /// # Timing model
- /// Custom, the timing model is whatever is returned as the Duration from `routine`.
- ///
- /// # Example
- /// ```rust
- /// #[macro_use] extern crate criterion;
- /// use criterion::*;
- /// use criterion::black_box;
- /// use std::time::Instant;
- ///
- /// fn foo() {
- /// // ...
- /// }
- ///
- /// fn bench(c: &mut Criterion) {
- /// c.bench_function("iter", move |b| {
- /// b.iter_custom(|iters| {
- /// let start = Instant::now();
- /// for _i in 0..iters {
- /// black_box(foo());
- /// }
- /// start.elapsed()
- /// })
- /// });
- /// }
- ///
- /// criterion_group!(benches, bench);
- /// criterion_main!(benches);
- /// ```
- ///
- #[inline(never)]
- pub fn iter_custom<R>(&mut self, mut routine: R)
- where
- R: FnMut(u64) -> M::Value,
- {
- self.iterated = true;
- let time_start = Instant::now();
- self.value = routine(self.iters);
- self.elapsed_time = time_start.elapsed();
- }
-
- #[doc(hidden)]
- pub fn iter_with_setup<I, O, S, R>(&mut self, setup: S, routine: R)
- where
- S: FnMut() -> I,
- R: FnMut(I) -> O,
- {
- self.iter_batched(setup, routine, BatchSize::PerIteration);
- }
-
- /// Times a `routine` by collecting its output on each iteration. This avoids timing the
- /// destructor of the value returned by `routine`.
- ///
- /// WARNING: This requires `O(iters * mem::size_of::<O>())` of memory, and `iters` is not under the
- /// control of the caller. If this causes out-of-memory errors, use `iter_batched` instead.
- ///
- /// # Timing model
- ///
- /// ``` text
- /// elapsed = Instant::now + iters * (routine) + Iterator::collect::<Vec<_>>
- /// ```
- ///
- /// # Example
- ///
- /// ```rust
- /// #[macro_use] extern crate criterion;
- ///
- /// use criterion::*;
- ///
- /// fn create_vector() -> Vec<u64> {
- /// # vec![]
- /// // ...
- /// }
- ///
- /// fn bench(c: &mut Criterion) {
- /// c.bench_function("with_drop", move |b| {
- /// // This will avoid timing the Vec::drop.
- /// b.iter_with_large_drop(|| create_vector())
- /// });
- /// }
- ///
- /// criterion_group!(benches, bench);
- /// criterion_main!(benches);
- /// ```
- ///
- pub fn iter_with_large_drop<O, R>(&mut self, mut routine: R)
- where
- R: FnMut() -> O,
- {
- self.iter_batched(|| (), |_| routine(), BatchSize::SmallInput);
- }
-
- #[doc(hidden)]
- pub fn iter_with_large_setup<I, O, S, R>(&mut self, setup: S, routine: R)
- where
- S: FnMut() -> I,
- R: FnMut(I) -> O,
- {
- self.iter_batched(setup, routine, BatchSize::NumBatches(1));
- }
-
- /// Times a `routine` that requires some input by generating a batch of input, then timing the
- /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for
- /// details on choosing the batch size. Use this when the routine must consume its input.
- ///
- /// For example, use this loop to benchmark sorting algorithms, because they require unsorted
- /// data on each iteration.
- ///
- /// # Timing model
- ///
- /// ```text
- /// elapsed = (Instant::now * num_batches) + (iters * (routine + O::drop)) + Vec::extend
- /// ```
- ///
- /// # Example
- ///
- /// ```rust
- /// #[macro_use] extern crate criterion;
- ///
- /// use criterion::*;
- ///
- /// fn create_scrambled_data() -> Vec<u64> {
- /// # vec![]
- /// // ...
- /// }
- ///
- /// // The sorting algorithm to test
- /// fn sort(data: &mut [u64]) {
- /// // ...
- /// }
- ///
- /// fn bench(c: &mut Criterion) {
- /// let data = create_scrambled_data();
- ///
- /// c.bench_function("with_setup", move |b| {
- /// // This will avoid timing the to_vec call.
- /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput)
- /// });
- /// }
- ///
- /// criterion_group!(benches, bench);
- /// criterion_main!(benches);
- /// ```
- ///
- #[inline(never)]
- pub fn iter_batched<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize)
- where
- S: FnMut() -> I,
- R: FnMut(I) -> O,
- {
- self.iterated = true;
- let batch_size = size.iters_per_batch(self.iters);
- assert!(batch_size != 0, "Batch size must not be zero.");
- let time_start = Instant::now();
- self.value = self.measurement.zero();
-
- if batch_size == 1 {
- for _ in 0..self.iters {
- let input = black_box(setup());
-
- let start = self.measurement.start();
- let output = routine(input);
- let end = self.measurement.end(start);
- self.value = self.measurement.add(&self.value, &end);
-
- drop(black_box(output));
- }
- } else {
- let mut iteration_counter = 0;
-
- while iteration_counter < self.iters {
- let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter);
-
- let inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>());
- let mut outputs = Vec::with_capacity(batch_size as usize);
-
- let start = self.measurement.start();
- outputs.extend(inputs.into_iter().map(&mut routine));
- let end = self.measurement.end(start);
- self.value = self.measurement.add(&self.value, &end);
-
- black_box(outputs);
-
- iteration_counter += batch_size;
- }
- }
-
- self.elapsed_time = time_start.elapsed();
- }
-
- /// Times a `routine` that requires some input by generating a batch of input, then timing the
- /// iteration of the benchmark over the input. See [`BatchSize`](enum.BatchSize.html) for
- /// details on choosing the batch size. Use this when the routine should accept the input by
- /// mutable reference.
- ///
- /// For example, use this loop to benchmark sorting algorithms, because they require unsorted
- /// data on each iteration.
- ///
- /// # Timing model
- ///
- /// ```text
- /// elapsed = (Instant::now * num_batches) + (iters * routine) + Vec::extend
- /// ```
- ///
- /// # Example
- ///
- /// ```rust
- /// #[macro_use] extern crate criterion;
- ///
- /// use criterion::*;
- ///
- /// fn create_scrambled_data() -> Vec<u64> {
- /// # vec![]
- /// // ...
- /// }
- ///
- /// // The sorting algorithm to test
- /// fn sort(data: &mut [u64]) {
- /// // ...
- /// }
- ///
- /// fn bench(c: &mut Criterion) {
- /// let data = create_scrambled_data();
- ///
- /// c.bench_function("with_setup", move |b| {
- /// // This will avoid timing the to_vec call.
- /// b.iter_batched(|| data.clone(), |mut data| sort(&mut data), BatchSize::SmallInput)
- /// });
- /// }
- ///
- /// criterion_group!(benches, bench);
- /// criterion_main!(benches);
- /// ```
- ///
- #[inline(never)]
- pub fn iter_batched_ref<I, O, S, R>(&mut self, mut setup: S, mut routine: R, size: BatchSize)
- where
- S: FnMut() -> I,
- R: FnMut(&mut I) -> O,
- {
- self.iterated = true;
- let batch_size = size.iters_per_batch(self.iters);
- assert!(batch_size != 0, "Batch size must not be zero.");
- let time_start = Instant::now();
- self.value = self.measurement.zero();
-
- if batch_size == 1 {
- for _ in 0..self.iters {
- let mut input = black_box(setup());
-
- let start = self.measurement.start();
- let output = routine(&mut input);
- let end = self.measurement.end(start);
- self.value = self.measurement.add(&self.value, &end);
-
- drop(black_box(output));
- drop(black_box(input));
- }
- } else {
- let mut iteration_counter = 0;
-
- while iteration_counter < self.iters {
- let batch_size = ::std::cmp::min(batch_size, self.iters - iteration_counter);
-
- let mut inputs = black_box((0..batch_size).map(|_| setup()).collect::<Vec<_>>());
- let mut outputs = Vec::with_capacity(batch_size as usize);
-
- let start = self.measurement.start();
- outputs.extend(inputs.iter_mut().map(&mut routine));
- let end = self.measurement.end(start);
- self.value = self.measurement.add(&self.value, &end);
-
- black_box(outputs);
-
- iteration_counter += batch_size;
- }
- }
- self.elapsed_time = time_start.elapsed();
- }
-
- // Benchmarks must actually call one of the iter methods. This causes benchmarks to fail loudly
- // if they don't.
- fn assert_iterated(&mut self) {
- if !self.iterated {
- panic!("Benchmark function must call Bencher::iter or related method.");
- }
- self.iterated = false;
- }
-}
-
/// Baseline describes how the baseline_directory is handled.
#[derive(Debug, Clone, Copy)]
pub enum Baseline {
@@ -664,15 +316,23 @@
/// is not installed.
Plotters,
}
+impl PlottingBackend {
+ fn create_plotter(&self) -> Box<dyn Plotter> {
+ match self {
+ PlottingBackend::Gnuplot => Box::new(Gnuplot::default()),
+ PlottingBackend::Plotters => Box::new(PlottersBackend::default()),
+ }
+ }
+}
#[derive(Debug, Clone)]
/// Enum representing the execution mode.
pub(crate) enum Mode {
- /// Run benchmarks normally
+ /// Run benchmarks normally.
Benchmark,
/// List all benchmarks but do not run them.
List,
- /// Run bennchmarks once to verify that they work, but otherwise do not measure them.
+ /// Run benchmarks once to verify that they work, but otherwise do not measure them.
Test,
/// Iterate benchmarks for a given length of time but do not analyze or report on them.
Profile(Duration),
@@ -696,16 +356,14 @@
/// the new load
/// - **Measurement**: The routine is repeatedly executed, and timing information is collected into
/// a sample
-/// - **Analysis**: The sample is analyzed and distiled into meaningful statistics that get
+/// - **Analysis**: The sample is analyzed and distilled into meaningful statistics that get
/// reported to stdout, stored in files, and plotted
/// - **Comparison**: The current sample is compared with the sample obtained in the previous
/// benchmark.
pub struct Criterion<M: Measurement = WallTime> {
config: BenchmarkConfig,
- plotting_backend: PlottingBackend,
- plotting_enabled: bool,
filter: Option<Regex>,
- report: Box<dyn Report>,
+ report: Reports,
output_directory: PathBuf,
baseline_directory: String,
baseline: Baseline,
@@ -718,6 +376,26 @@
mode: Mode,
}
+/// Returns the Cargo target directory, possibly calling `cargo metadata` to
+/// figure it out.
+fn cargo_target_directory() -> Option<PathBuf> {
+ #[derive(Deserialize)]
+ struct Metadata {
+ target_directory: PathBuf,
+ }
+
+ env::var_os("CARGO_TARGET_DIR")
+ .map(PathBuf::from)
+ .or_else(|| {
+ let output = Command::new(env::var_os("CARGO")?)
+ .args(&["metadata", "--format-version", "1"])
+ .output()
+ .ok()?;
+ let metadata: Metadata = serde_json::from_slice(&output.stdout).ok()?;
+ Some(metadata.target_directory)
+ })
+}
+
impl Default for Criterion {
/// Creates a benchmark manager with the following default settings:
///
@@ -731,25 +409,18 @@
/// - Plotting: enabled, using gnuplot if available or plotters if gnuplot is not available
/// - No filter
fn default() -> Criterion {
- let mut reports: Vec<Box<dyn Report>> = vec![];
- if CARGO_CRITERION_CONNECTION.is_none() {
- reports.push(Box::new(CliReport::new(false, false, false)));
- }
- reports.push(Box::new(FileCsvReport));
-
- // Set criterion home to (in descending order of preference):
- // - $CRITERION_HOME (cargo-criterion sets this, but other users could as well)
- // - $CARGO_TARGET_DIR/criterion
- // - ./target/criterion
- let output_directory = if let Some(value) = std::env::var_os("CRITERION_HOME") {
- PathBuf::from(value)
- } else if let Some(value) = std::env::var_os("CARGO_TARGET_DIR") {
- PathBuf::from(value).join("criterion")
- } else {
- PathBuf::from("target/criterion")
+ let reports = Reports {
+ cli_enabled: true,
+ cli: CliReport::new(false, false, false),
+ bencher_enabled: false,
+ bencher: BencherReport,
+ html_enabled: true,
+ html: Html::new(DEFAULT_PLOTTING_BACKEND.create_plotter()),
+ csv_enabled: true,
+ csv: FileCsvReport,
};
- Criterion {
+ let mut criterion = Criterion {
config: BenchmarkConfig {
confidence_level: 0.95,
measurement_time: Duration::new(5, 0),
@@ -760,14 +431,12 @@
warm_up_time: Duration::new(3, 0),
sampling_mode: SamplingMode::Auto,
},
- plotting_backend: *DEFAULT_PLOTTING_BACKEND,
- plotting_enabled: true,
filter: None,
- report: Box::new(Reports::new(reports)),
+ report: reports,
baseline_directory: "base".to_owned(),
baseline: Baseline::Save,
load_baseline: None,
- output_directory,
+ output_directory: DEFAULT_OUTPUT_DIRECTORY.clone(),
all_directories: HashSet::new(),
all_titles: HashSet::new(),
measurement: WallTime,
@@ -776,7 +445,16 @@
.as_ref()
.map(|mtx| mtx.lock().unwrap()),
mode: Mode::Benchmark,
+ };
+
+ if criterion.connection.is_some() {
+ // disable all reports when connected to cargo-criterion; it will do the reporting.
+ criterion.report.cli_enabled = false;
+ criterion.report.bencher_enabled = false;
+ criterion.report.csv_enabled = false;
+ criterion.report.html_enabled = false;
}
+ criterion
}
}
@@ -787,8 +465,6 @@
// Can't use struct update syntax here because they're technically different types.
Criterion {
config: self.config,
- plotting_backend: self.plotting_backend,
- plotting_enabled: self.plotting_enabled,
filter: self.filter,
report: self.report,
baseline_directory: self.baseline_directory,
@@ -817,17 +493,15 @@
/// if not.
///
/// Panics if `backend` is `PlottingBackend::Gnuplot` and gnuplot is not available.
- pub fn plotting_backend(self, backend: PlottingBackend) -> Criterion<M> {
+ pub fn plotting_backend(mut self, backend: PlottingBackend) -> Criterion<M> {
if let PlottingBackend::Gnuplot = backend {
if GNUPLOT_VERSION.is_err() {
panic!("Gnuplot plotting backend was requested, but gnuplot is not available. To continue, either install Gnuplot or allow Criterion.rs to fall back to using plotters.");
}
}
- Criterion {
- plotting_backend: backend,
- ..self
- }
+ self.report.html = Html::new(backend.create_plotter());
+ self
}
/// Changes the default size of the sample for benchmarks run with this runner.
@@ -944,7 +618,7 @@
/// noise.
///
/// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
- /// the statistical robustness against noise, but it also weaken's Criterion.rs' ability to
+ /// the statistical robustness against noise, but it also weakens Criterion.rs' ability to
/// detect small but real changes in the performance. By setting the significance level
/// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
/// report more spurious differences.
@@ -961,40 +635,26 @@
self
}
- fn create_plotter(&self) -> Box<dyn Plotter> {
- match self.plotting_backend {
- PlottingBackend::Gnuplot => Box::new(Gnuplot::default()),
- PlottingBackend::Plotters => Box::new(PlottersBackend::default()),
- }
- }
-
/// Enables plotting
pub fn with_plots(mut self) -> Criterion<M> {
- self.plotting_enabled = true;
- let mut reports: Vec<Box<dyn Report>> = vec![];
+ // If running under cargo-criterion then don't re-enable the reports; let it do the reporting.
if self.connection.is_none() {
- reports.push(Box::new(CliReport::new(false, false, false)));
+ self.report.html_enabled = true;
}
- reports.push(Box::new(FileCsvReport));
- reports.push(Box::new(Html::new(self.create_plotter())));
- self.report = Box::new(Reports::new(reports));
-
self
}
/// Disables plotting
pub fn without_plots(mut self) -> Criterion<M> {
- self.plotting_enabled = false;
- let mut reports: Vec<Box<dyn Report>> = vec![];
- if self.connection.is_none() {
- reports.push(Box::new(CliReport::new(false, false, false)));
- }
- reports.push(Box::new(FileCsvReport));
- self.report = Box::new(Reports::new(reports));
+ self.report.html_enabled = false;
self
}
/// Return true if generation of the plots is possible.
+ #[deprecated(
+ since = "0.3.4",
+ note = "No longer useful; since the plotters backend is available Criterion.rs can always generate plots"
+ )]
pub fn can_plot(&self) -> bool {
// Trivially true now that we have plotters.
// TODO: Deprecate and remove this.
@@ -1030,6 +690,13 @@
self
}
+ /// Override whether the CLI output will be colored or not. Usually you would use the `--color`
+ /// CLI argument, but this is available for programmmatic use as well.
+ pub fn with_output_color(mut self, enabled: bool) -> Criterion<M> {
+ self.report.cli.enable_text_coloring = enabled;
+ self
+ }
+
/// Set the output directory (currently for testing only)
#[doc(hidden)]
pub fn output_directory(mut self, path: &Path) -> Criterion<M> {
@@ -1179,6 +846,9 @@
scripts alongside the generated plots.
To test that the benchmarks work, run `cargo test --benches`
+
+NOTE: If you see an 'unrecognized option' error using any of the options above, see:
+https://bheisler.github.io/criterion.rs/book/faq.html
")
.get_matches();
@@ -1277,10 +947,16 @@
if self.connection.is_some() {
// disable all reports when connected to cargo-criterion; it will do the reporting.
- self.report = Box::new(Reports::new(vec![]));
+ self.report.cli_enabled = false;
+ self.report.bencher_enabled = false;
+ self.report.csv_enabled = false;
+ self.report.html_enabled = false;
} else {
- let cli_report: Box<dyn Report> = match matches.value_of("output-format") {
- Some("bencher") => Box::new(BencherReport),
+ match matches.value_of("output-format") {
+ Some("bencher") => {
+ self.report.bencher_enabled = true;
+ self.report.cli_enabled = false;
+ }
_ => {
let verbose = matches.is_present("verbose");
let stdout_isatty = atty::is(atty::Stream::Stdout);
@@ -1296,21 +972,12 @@
}
_ => enable_text_coloring = stdout_isatty,
};
- Box::new(CliReport::new(
- enable_text_overwrite,
- enable_text_coloring,
- verbose,
- ))
+ self.report.bencher_enabled = false;
+ self.report.cli_enabled = true;
+ self.report.cli =
+ CliReport::new(enable_text_overwrite, enable_text_coloring, verbose);
}
};
-
- let mut reports: Vec<Box<dyn Report>> = vec![];
- reports.push(cli_report);
- reports.push(Box::new(FileCsvReport));
- if self.plotting_enabled {
- reports.push(Box::new(Html::new(self.create_plotter())));
- }
- self.report = Box::new(Reports::new(reports));
}
if let Some(dir) = matches.value_of("load-baseline") {
@@ -1502,9 +1169,14 @@
where
F: FnMut(&mut Bencher<'_, M>, &I),
{
- // Guaranteed safe because external callers can't create benchmark IDs without a function
- // name or parameter
- let group_name = id.function_name.unwrap();
+ // It's possible to use BenchmarkId::from_parameter to create a benchmark ID with no function
+ // name. That's intended for use with BenchmarkGroups where the function name isn't necessary,
+ // but here it is.
+ let group_name = id.function_name.expect(
+ "Cannot use BenchmarkId::from_parameter with Criterion::bench_with_input. \
+ Consider using a BenchmarkGroup or BenchmarkId::new instead.",
+ );
+ // Guaranteed safe because external callers can't create benchmark IDs without a parameter
let parameter = id.parameter.unwrap();
self.benchmark_group(group_name).bench_with_input(
BenchmarkId::no_function_with_input(parameter),
@@ -1537,7 +1209,9 @@
/// criterion_group!(benches, bench);
/// criterion_main!(benches);
/// ```
- #[doc(hidden)] // Soft-deprecated, use benchmark groups instead
+ #[doc(hidden)]
+ #[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")]
+ #[allow(deprecated)]
pub fn bench_function_over_inputs<I, F>(
&mut self,
id: &str,
@@ -1588,7 +1262,9 @@
/// criterion_group!(benches, bench);
/// criterion_main!(benches);
/// ```
- #[doc(hidden)] // Soft-deprecated, use benchmark groups instead
+ #[doc(hidden)]
+ #[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")]
+ #[allow(deprecated)]
pub fn bench_functions<I>(
&mut self,
id: &str,
@@ -1631,7 +1307,8 @@
/// criterion_group!(benches, bench);
/// criterion_main!(benches);
/// ```
- #[doc(hidden)] // Soft-deprecated, use benchmark groups instead
+ #[doc(hidden)]
+ #[deprecated(since = "0.3.4", note = "Please use BenchmarkGroups instead.")]
pub fn bench<B: BenchmarkDefinition<M>>(
&mut self,
group_id: &str,
@@ -1786,7 +1463,7 @@
let m_ns = target_time.to_nanos();
// Solve: [d + 2*d + 3*d + ... + n*d] * met = m_ns
let total_runs = n * (n + 1) / 2;
- let d = (m_ns as f64 / met / total_runs as f64).ceil() as u64;
+ let d = ((m_ns as f64 / met / total_runs as f64).ceil() as u64).max(1);
let expected_ns = total_runs as f64 * d as f64 * met;
if d == 1 {
@@ -1802,7 +1479,7 @@
recommended_sample_size
);
} else {
- println!("or enable flat sampling.");
+ println!(" or enable flat sampling.");
}
}
@@ -1814,7 +1491,7 @@
let m_ns = target_time.to_nanos() as f64;
let time_per_sample = m_ns / (n as f64);
// This is pretty simplistic; we could do something smarter to fit into the allotted time.
- let iterations_per_sample = (time_per_sample / met).ceil() as u64;
+ let iterations_per_sample = ((time_per_sample / met).ceil() as u64).max(1);
let expected_ns = met * (iterations_per_sample * n) as f64;
@@ -1897,3 +1574,53 @@
}
Criterion::default().configure_from_args().final_summary();
}
+
+/// Print a warning informing users about upcoming changes to features
+#[cfg(not(feature = "html_reports"))]
+#[doc(hidden)]
+pub fn __warn_about_html_reports_feature() {
+ if CARGO_CRITERION_CONNECTION.is_none() {
+ println!(
+ "WARNING: HTML report generation will become a non-default optional feature in Criterion.rs 0.4.0."
+ );
+ println!(
+ "This feature is being moved to cargo-criterion \
+ (https://github.com/bheisler/cargo-criterion) and will be optional in a future \
+ version of Criterion.rs. To silence this warning, either switch to cargo-criterion or \
+ enable the 'html_reports' feature in your Cargo.toml."
+ );
+ println!();
+ }
+}
+
+/// Print a warning informing users about upcoming changes to features
+#[cfg(feature = "html_reports")]
+#[doc(hidden)]
+pub fn __warn_about_html_reports_feature() {
+ // They have the feature enabled, so they're ready for the update.
+}
+
+/// Print a warning informing users about upcoming changes to features
+#[cfg(not(feature = "cargo_bench_support"))]
+#[doc(hidden)]
+pub fn __warn_about_cargo_bench_support_feature() {
+ if CARGO_CRITERION_CONNECTION.is_none() {
+ println!(
+ "WARNING: In Criterion.rs 0.4.0, running criterion benchmarks outside of cargo-criterion will become a default optional feature."
+ );
+ println!(
+ "The statistical analysis and reporting is being moved to cargo-criterion \
+ (https://github.com/bheisler/cargo-criterion) and will be optional in a future \
+ version of Criterion.rs. To silence this warning, either switch to cargo-criterion or \
+ enable the 'cargo_bench_support' feature in your Cargo.toml."
+ );
+ println!();
+ }
+}
+
+/// Print a warning informing users about upcoming changes to features
+#[cfg(feature = "cargo_bench_support")]
+#[doc(hidden)]
+pub fn __warn_about_cargo_bench_support_feature() {
+ // They have the feature enabled, so they're ready for the update.
+}
diff --git a/src/macros.rs b/src/macros.rs
index ca8bb5c..2ca7a7d 100755
--- a/src/macros.rs
+++ b/src/macros.rs
@@ -120,6 +120,9 @@
macro_rules! criterion_main {
( $( $group:path ),+ $(,)* ) => {
fn main() {
+ $crate::__warn_about_html_reports_feature();
+ $crate::__warn_about_cargo_bench_support_feature();
+
$(
$group();
)+
diff --git a/src/measurement.rs b/src/measurement.rs
index 8c6d708..e253670 100755
--- a/src/measurement.rs
+++ b/src/measurement.rs
@@ -15,7 +15,7 @@
/// nanoseconds, the values passed to the formatter will be in nanoseconds).
///
/// Implementors are encouraged to format the values in a way that is intuitive for humans and
-/// uses the SI prefix system. For example, the format used by [WallTime](struct.Walltime.html)
+/// uses the SI prefix system. For example, the format used by [WallTime](struct.WallTime.html)
/// can display the value in units ranging from picoseconds to seconds depending on the magnitude
/// of the elapsed time in nanoseconds.
pub trait ValueFormatter {
diff --git a/src/plot/gnuplot_backend/summary.rs b/src/plot/gnuplot_backend/summary.rs
index cdd0604..e944360 100755
--- a/src/plot/gnuplot_backend/summary.rs
+++ b/src/plot/gnuplot_backend/summary.rs
@@ -174,6 +174,7 @@
.configure(Axis::BottomX, |a| {
a.configure(Grid::Major, |g| g.show())
.configure(Grid::Minor, |g| g.hide())
+ .set(Range::Limits(0., max as f64 * one[0]))
.set(Label(format!("Average time ({})", unit)))
.set(axis_scale.to_gnuplot())
})
@@ -200,9 +201,9 @@
if is_first {
is_first = false;
- c.set(DARK_BLUE).set(Label("PDF")).set(Opacity(0.25))
+ c.set(DARK_BLUE).set(Label("PDF"))
} else {
- c.set(DARK_BLUE).set(Opacity(0.25))
+ c.set(DARK_BLUE)
}
});
}
diff --git a/src/plot/plotters_backend/distributions.rs b/src/plot/plotters_backend/distributions.rs
index ed4d61a..82f9eae 100755
--- a/src/plot/plotters_backend/distributions.rs
+++ b/src/plot/plotters_backend/distributions.rs
@@ -69,7 +69,7 @@
)
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(x_range, y_range)
+ .build_cartesian_2d(x_range, y_range)
.unwrap();
chart
@@ -224,7 +224,7 @@
)
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(x_min..x_max, y_range.clone())
+ .build_cartesian_2d(x_min..x_max, y_range.clone())
.unwrap();
chart
diff --git a/src/plot/plotters_backend/iteration_times.rs b/src/plot/plotters_backend/iteration_times.rs
index a2204df..4d0a22a 100755
--- a/src/plot/plotters_backend/iteration_times.rs
+++ b/src/plot/plotters_backend/iteration_times.rs
@@ -30,14 +30,14 @@
.margin((5).percent())
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(x_range, y_range)
+ .build_cartesian_2d(x_range, y_range)
.unwrap();
chart
.configure_mesh()
.y_desc(format!("Average Iteration Time ({})", unit))
.x_label_formatter(&|x| pretty_print_float(*x, true))
- .line_style_2(&TRANSPARENT)
+ .light_line_style(&TRANSPARENT)
.draw()
.unwrap();
@@ -97,14 +97,14 @@
.margin((5).percent())
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(0.0..max_samples, y_range)
+ .build_cartesian_2d(0.0..max_samples, y_range)
.unwrap();
chart
.configure_mesh()
.y_desc(format!("Average Iteration Time ({})", unit))
.x_label_formatter(&|x| pretty_print_float(*x, true))
- .line_style_2(&TRANSPARENT)
+ .light_line_style(&TRANSPARENT)
.draw()
.unwrap();
diff --git a/src/plot/plotters_backend/pdf.rs b/src/plot/plotters_backend/pdf.rs
index d8f3541..1297b79 100755
--- a/src/plot/plotters_backend/pdf.rs
+++ b/src/plot/plotters_backend/pdf.rs
@@ -50,7 +50,7 @@
.margin((5).percent())
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(x_range, y_range.clone())
+ .build_cartesian_2d(x_range, y_range.clone())
.unwrap();
chart
@@ -138,7 +138,7 @@
.margin((5).percent())
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(xs_.min()..xs_.max(), 0.0..y_limit)
+ .build_cartesian_2d(xs_.min()..xs_.max(), 0.0..y_limit)
.unwrap();
chart
@@ -218,7 +218,7 @@
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Right, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(xs_.min()..xs_.max(), 0.0..max_iters)
+ .build_cartesian_2d(xs_.min()..xs_.max(), 0.0..max_iters)
.unwrap()
.set_secondary_coord(xs_.min()..xs_.max(), 0.0..range.end);
diff --git a/src/plot/plotters_backend/regression.rs b/src/plot/plotters_backend/regression.rs
index 54bffa6..c944dbb 100755
--- a/src/plot/plotters_backend/regression.rs
+++ b/src/plot/plotters_backend/regression.rs
@@ -53,7 +53,7 @@
.margin((5).percent())
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(x_range, y_range)
+ .build_cartesian_2d(x_range, y_range)
.unwrap();
chart
@@ -61,7 +61,7 @@
.x_desc(x_label)
.y_desc(format!("Total sample time ({})", unit))
.x_label_formatter(&|x| pretty_print_float(x * x_scale, true))
- .line_style_2(&TRANSPARENT)
+ .light_line_style(&TRANSPARENT)
.draw()
.unwrap();
@@ -179,7 +179,7 @@
.margin((5).percent())
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(0.0..max_iters, 0.0..y_max)
+ .build_cartesian_2d(0.0..max_iters, 0.0..y_max)
.unwrap();
chart
@@ -187,7 +187,7 @@
.x_desc(x_label)
.y_desc(format!("Total sample time ({})", unit))
.x_label_formatter(&|x| pretty_print_float(x * x_scale, true))
- .line_style_2(&TRANSPARENT)
+ .light_line_style(&TRANSPARENT)
.draw()
.unwrap();
diff --git a/src/plot/plotters_backend/summary.rs b/src/plot/plotters_backend/summary.rs
index dd02d0c..14f4b5e 100755
--- a/src/plot/plotters_backend/summary.rs
+++ b/src/plot/plotters_backend/summary.rs
@@ -1,7 +1,10 @@
use super::*;
use crate::AxisScale;
use itertools::Itertools;
-use plotters::coord::{AsRangedCoord, Shift};
+use plotters::coord::{
+ ranged1d::{AsRangedCoord, ValueFormatter as PlottersValueFormatter},
+ Shift,
+};
use std::cmp::Ordering;
use std::path::Path;
@@ -25,7 +28,7 @@
value_type: ValueType,
axis_scale: AxisScale,
) {
- let (unit, series_data) = line_comparision_series_data(formatter, all_curves);
+ let (unit, series_data) = line_comparison_series_data(formatter, all_curves);
let x_range =
plotters::data::fitting_range(series_data.iter().map(|(_, xs, _)| xs.iter()).flatten());
@@ -33,7 +36,7 @@
plotters::data::fitting_range(series_data.iter().map(|(_, _, ys)| ys.iter()).flatten());
let root_area = SVGBackend::new(&path, SIZE)
.into_drawing_area()
- .titled(&format!("{}: Comparision", title), (DEFAULT_FONT, 20))
+ .titled(&format!("{}: Comparison", title), (DEFAULT_FONT, 20))
.unwrap();
match axis_scale {
@@ -58,7 +61,10 @@
y_range: YR,
value_type: ValueType,
data: Vec<(Option<&String>, Vec<f64>, Vec<f64>)>,
-) {
+) where
+ XR::CoordDescType: PlottersValueFormatter<f64>,
+ YR::CoordDescType: PlottersValueFormatter<f64>,
+{
let input_suffix = match value_type {
ValueType::Bytes => " Size (Bytes)",
ValueType::Elements => " Size (Elements)",
@@ -69,7 +75,7 @@
.margin((5).percent())
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(x_range, y_range)
+ .build_cartesian_2d(x_range, y_range)
.unwrap();
chart
@@ -109,7 +115,7 @@
}
#[allow(clippy::type_complexity)]
-fn line_comparision_series_data<'a>(
+fn line_comparison_series_data<'a>(
formatter: &dyn ValueFormatter,
all_curves: &[&(&'a BenchmarkId, Vec<f64>)],
) -> (&'static str, Vec<(Option<&'a String>, Vec<f64>, Vec<f64>)>) {
@@ -190,7 +196,9 @@
formatter.scale_values(max, xs);
});
- let x_range = plotters::data::fitting_range(kdes.iter().map(|(_, xs, _)| xs.iter()).flatten());
+ let mut x_range =
+ plotters::data::fitting_range(kdes.iter().map(|(_, xs, _)| xs.iter()).flatten());
+ x_range.start = 0.0;
let y_range = -0.5..all_curves.len() as f64 - 0.5;
let size = (960, 150 + (18 * all_curves.len() as u32));
@@ -215,12 +223,15 @@
x_range: XR,
y_range: YR,
data: Vec<(&str, Box<[f64]>, Box<[f64]>)>,
-) {
+) where
+ XR::CoordDescType: PlottersValueFormatter<f64>,
+ YR::CoordDescType: PlottersValueFormatter<f64>,
+{
let mut chart = ChartBuilder::on(&root_area)
.margin((5).percent())
.set_label_area_size(LabelAreaPosition::Left, (10).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_width().min(40))
- .build_ranged(x_range, y_range)
+ .build_cartesian_2d(x_range, y_range)
.unwrap();
chart
@@ -241,7 +252,7 @@
.draw_series(AreaSeries::new(
x.iter().zip(y.iter()).map(|(x, y)| (*x, base + *y / 2.0)),
base,
- &DARK_BLUE.mix(0.25),
+ &DARK_BLUE,
))
.unwrap();
@@ -249,7 +260,7 @@
.draw_series(AreaSeries::new(
x.iter().zip(y.iter()).map(|(x, y)| (*x, base - *y / 2.0)),
base,
- &DARK_BLUE.mix(0.25),
+ &DARK_BLUE,
))
.unwrap();
}
diff --git a/src/plot/plotters_backend/t_test.rs b/src/plot/plotters_backend/t_test.rs
index 741cc9e..d9c1508 100755
--- a/src/plot/plotters_backend/t_test.rs
+++ b/src/plot/plotters_backend/t_test.rs
@@ -23,7 +23,7 @@
.caption(format!("{}: Welch t test", title), (DEFAULT_FONT, 20))
.set_label_area_size(LabelAreaPosition::Left, (5).percent_width().min(60))
.set_label_area_size(LabelAreaPosition::Bottom, (5).percent_height().min(40))
- .build_ranged(x_range, y_range.clone())
+ .build_cartesian_2d(x_range, y_range.clone())
.unwrap();
chart
diff --git a/src/report.rs b/src/report.rs
index 216904f..9148a9e 100755
--- a/src/report.rs
+++ b/src/report.rs
@@ -1,6 +1,6 @@
-use crate::stats::bivariate::regression::Slope;
-use crate::stats::bivariate::Data;
use crate::stats::univariate::outliers::tukey::LabeledSample;
+use crate::{csv_report::FileCsvReport, stats::bivariate::regression::Slope};
+use crate::{html::Html, stats::bivariate::Data};
use crate::estimate::{ChangeDistributions, ChangeEstimates, Distributions, Estimate, Estimates};
use crate::format;
@@ -299,101 +299,68 @@
}
pub(crate) struct Reports {
- reports: Vec<Box<dyn Report>>,
+ pub(crate) cli_enabled: bool,
+ pub(crate) cli: CliReport,
+ pub(crate) bencher_enabled: bool,
+ pub(crate) bencher: BencherReport,
+ pub(crate) csv_enabled: bool,
+ pub(crate) csv: FileCsvReport,
+ pub(crate) html_enabled: bool,
+ pub(crate) html: Html,
}
-impl Reports {
- pub fn new(reports: Vec<Box<dyn Report>>) -> Reports {
- Reports { reports }
- }
+macro_rules! reports_impl {
+ (fn $name:ident(&self, $($argn:ident: $argt:ty),*)) => {
+ fn $name(&self, $($argn: $argt),* ) {
+ if self.cli_enabled {
+ self.cli.$name($($argn),*);
+ }
+ if self.bencher_enabled {
+ self.bencher.$name($($argn),*);
+ }
+ if self.csv_enabled {
+ self.csv.$name($($argn),*);
+ }
+ if self.html_enabled {
+ self.html.$name($($argn),*);
+ }
+ }
+ };
}
+
impl Report for Reports {
- fn test_start(&self, id: &BenchmarkId, context: &ReportContext) {
- for report in &self.reports {
- report.test_start(id, context);
- }
- }
- fn test_pass(&self, id: &BenchmarkId, context: &ReportContext) {
- for report in &self.reports {
- report.test_pass(id, context);
- }
- }
-
- fn benchmark_start(&self, id: &BenchmarkId, context: &ReportContext) {
- for report in &self.reports {
- report.benchmark_start(id, context);
- }
- }
-
- fn profile(&self, id: &BenchmarkId, context: &ReportContext, profile_ns: f64) {
- for report in &self.reports {
- report.profile(id, context, profile_ns);
- }
- }
-
- fn warmup(&self, id: &BenchmarkId, context: &ReportContext, warmup_ns: f64) {
- for report in &self.reports {
- report.warmup(id, context, warmup_ns);
- }
- }
-
- fn terminated(&self, id: &BenchmarkId, context: &ReportContext) {
- for report in &self.reports {
- report.terminated(id, context);
- }
- }
-
- fn analysis(&self, id: &BenchmarkId, context: &ReportContext) {
- for report in &self.reports {
- report.analysis(id, context);
- }
- }
-
- fn measurement_start(
+ reports_impl!(fn test_start(&self, id: &BenchmarkId, context: &ReportContext));
+ reports_impl!(fn test_pass(&self, id: &BenchmarkId, context: &ReportContext));
+ reports_impl!(fn benchmark_start(&self, id: &BenchmarkId, context: &ReportContext));
+ reports_impl!(fn profile(&self, id: &BenchmarkId, context: &ReportContext, profile_ns: f64));
+ reports_impl!(fn warmup(&self, id: &BenchmarkId, context: &ReportContext, warmup_ns: f64));
+ reports_impl!(fn terminated(&self, id: &BenchmarkId, context: &ReportContext));
+ reports_impl!(fn analysis(&self, id: &BenchmarkId, context: &ReportContext));
+ reports_impl!(fn measurement_start(
&self,
id: &BenchmarkId,
context: &ReportContext,
sample_count: u64,
estimate_ns: f64,
- iter_count: u64,
- ) {
- for report in &self.reports {
- report.measurement_start(id, context, sample_count, estimate_ns, iter_count);
- }
- }
-
+ iter_count: u64
+ ));
+ reports_impl!(
fn measurement_complete(
&self,
id: &BenchmarkId,
context: &ReportContext,
measurements: &MeasurementData<'_>,
- formatter: &dyn ValueFormatter,
- ) {
- for report in &self.reports {
- report.measurement_complete(id, context, measurements, formatter);
- }
- }
-
+ formatter: &dyn ValueFormatter
+ ));
+ reports_impl!(
fn summarize(
&self,
context: &ReportContext,
all_ids: &[BenchmarkId],
- formatter: &dyn ValueFormatter,
- ) {
- for report in &self.reports {
- report.summarize(context, all_ids, formatter);
- }
- }
+ formatter: &dyn ValueFormatter
+ ));
- fn final_summary(&self, context: &ReportContext) {
- for report in &self.reports {
- report.final_summary(context);
- }
- }
- fn group_separator(&self) {
- for report in &self.reports {
- report.group_separator();
- }
- }
+ reports_impl!(fn final_summary(&self, context: &ReportContext));
+ reports_impl!(fn group_separator(&self, ));
}
pub(crate) struct CliReport {
@@ -428,7 +395,7 @@
}
}
- //Passing a String is the common case here.
+ // Passing a String is the common case here.
#[cfg_attr(feature = "cargo-clippy", allow(clippy::needless_pass_by_value))]
fn print_overwritable(&self, s: String) {
if self.enable_text_overwrite {
@@ -633,7 +600,7 @@
let point_estimate = mean_est.point_estimate;
let mut point_estimate_str = format::change(point_estimate, true);
// The change in throughput is related to the change in timing. Reducing the timing by
- // 50% increases the througput by 100%.
+ // 50% increases the throughput by 100%.
let to_thrpt_estimate = |ratio: f64| 1.0 / (1.0 + ratio) - 1.0;
let mut thrpt_point_estimate_str =
format::change(to_thrpt_estimate(point_estimate), true);
diff --git a/src/routine.rs b/src/routine.rs
index a54cdd6..5831415 100755
--- a/src/routine.rs
+++ b/src/routine.rs
@@ -1,7 +1,7 @@
use crate::benchmark::BenchmarkConfig;
use crate::connection::OutgoingMessage;
use crate::measurement::Measurement;
-use crate::report::{BenchmarkId, ReportContext};
+use crate::report::{BenchmarkId, Report, ReportContext};
use crate::{ActualSamplingMode, Bencher, Criterion, DurationExt};
use std::marker::PhantomData;
use std::time::Duration;
@@ -130,20 +130,22 @@
.map(|count| count as f64 * met)
.sum();
- criterion.report.measurement_start(
- id,
- report_context,
- n,
- expected_ns,
- m_iters.iter().sum(),
- );
+ // Use saturating_add to handle overflow.
+ let mut total_iters = 0u64;
+ for count in m_iters.iter().copied() {
+ total_iters = total_iters.saturating_add(count);
+ }
+
+ criterion
+ .report
+ .measurement_start(id, report_context, n, expected_ns, total_iters);
if let Some(conn) = &criterion.connection {
conn.send(&OutgoingMessage::MeasurementStart {
id: id.into(),
sample_count: n,
estimate_ns: expected_ns,
- iter_count: m_iters.iter().sum(),
+ iter_count: total_iters,
})
.unwrap();
}
@@ -234,7 +236,7 @@
return (elapsed_time.to_nanos(), total_iters);
}
- b.iters *= 2;
+ b.iters = b.iters.wrapping_mul(2);
}
}
}
diff --git a/tests/criterion_tests.rs b/tests/criterion_tests.rs
index 562eec9..cca448e 100755
--- a/tests/criterion_tests.rs
+++ b/tests/criterion_tests.rs
@@ -1,3 +1,5 @@
+#![allow(deprecated)]
+
use criterion;
use serde_json;
@@ -485,6 +487,15 @@
group.finish();
}
+#[test]
+fn test_criterion_doesnt_panic_if_measured_time_is_zero() {
+ let dir = temp_dir();
+ let mut c = short_benchmark(&dir);
+ c.bench_function("zero_time", |bencher| {
+ bencher.iter_custom(|_iters| Duration::new(0, 0))
+ });
+}
+
mod macros {
use super::{criterion, criterion_group, criterion_main};
