lib.rs - platform/external/rust/crates/bencher - Git at Google

 // Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT
 // file at the top-level directory of this distribution and at
 // http://rust-lang.org/COPYRIGHT.
 //
 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.

 //! Simplified stable-compatible benchmark runner.
 //!
 //! Almost all user code will only be interested in `Bencher` and the
 //! macros that are used to describe benchmarker functions and
 //! the benchmark runner.
 //!
 //! NOTE: There's no proper `black_box` yet in this stable port of the
 //! benchmark runner, only a workaround implementation. It may not work
 //! exactly like the upstream `test::black_box`.
 //!
 //! One way to use this crate is to use it as dev-dependency and setup
 //! cargo to compile a file in `benches/` that runs without the testing harness.
 //!
 //! In Cargo.toml:
 //!
 //! ```ignore
 //! [[bench]]
 //! name = "example"
 //! harness = false
 //! ```
 //!
 //! In benches/example.rs:
 //!
 //! ```
 //! #[macro_use]
 //! extern crate bencher;
 //!
 //! use bencher::Bencher;
 //!
 //! fn a(bench: &mut Bencher) {
 //!     bench.iter(|| {
 //!         (0..1000).fold(0, |x, y| x + y)
 //!     })
 //! }
 //!
 //! fn b(bench: &mut Bencher) {
 //!     const N: usize = 1024;
 //!     bench.iter(|| {
 //!         vec![0u8; N]
 //!     });
 //!
 //!     bench.bytes = N as u64;
 //! }
 //!
 //! benchmark_group!(benches, a, b);
 //! benchmark_main!(benches);
 //!
 //! # #[cfg(never)]
 //! # fn main() { }
 //! ```
 //!
 //! Use `cargo bench` as usual. A command line argument can be used to filter
 //! which benchmarks to run.

 pub use self::TestFn::*;
 use self::TestResult::*;
 use self::TestEvent::*;
 use self::NamePadding::*;
 use self::OutputLocation::*;

 use std::borrow::Cow;
 use std::cmp;
 use std::fmt;
 use std::fs::File;
 use std::io::prelude::*;
 use std::io;
 use std::iter::repeat;
 use std::mem::forget;
 use std::path::PathBuf;
 use std::ptr;
 use std::time::{Instant, Duration};

 pub mod stats;
 mod macros;

 // The name of a test. By convention this follows the rules for rust
 // paths; i.e. it should be a series of identifiers separated by double
 // colons. This way if some test runner wants to arrange the tests
 // hierarchically it may.

 pub type TestName = Cow<'static, str>;

 #[derive(Clone, Copy, PartialEq, Eq)]
 enum NamePadding {
     PadOnRight,
 }

 impl TestDesc {
     fn padded_name(&self, column_count: usize, align: NamePadding) -> String {
         let mut name = self.name.to_string();
         let fill = column_count.saturating_sub(name.len());
         let pad = repeat(" ").take(fill).collect::<String>();
         match align {
             PadOnRight => {
                 name.push_str(&pad);
                 name
             }
         }
     }
 }

 /// Represents a benchmark function.
 pub trait TDynBenchFn: Send {
     fn run(&self, harness: &mut Bencher);
 }

 // A function that runs a test. If the function returns successfully,
 // the test succeeds; if the function panics then the test fails. We
 // may need to come up with a more clever definition of test in order
 // to support isolation of tests into threads.
 pub enum TestFn {
     StaticBenchFn(fn(&mut Bencher)),
     DynBenchFn(Box<TDynBenchFn + 'static>),
 }

 impl TestFn {
     fn padding(&self) -> NamePadding {
         match *self {
             StaticBenchFn(..) |
             DynBenchFn(..) => PadOnRight,
         }
     }
 }

 impl fmt::Debug for TestFn {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         f.write_str(match *self {
             StaticBenchFn(..) => "StaticBenchFn(..)",
             DynBenchFn(..) => "DynBenchFn(..)",
         })
     }
 }

 /// Manager of the benchmarking runs.
 ///
 /// This is fed into functions marked with `#[bench]` to allow for
 /// set-up & tear-down before running a piece of code repeatedly via a
 /// call to `iter`.
 #[derive(Copy, Clone)]
 pub struct Bencher {
     iterations: u64,
     dur: Duration,
     pub bytes: u64,
 }

 // The definition of a single test. A test runner will run a list of
 // these.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct TestDesc {
     pub name: TestName,
     pub ignore: bool,
 }

 #[derive(Clone)]
 pub struct TestPaths {
     pub file: PathBuf,         // e.g., compile-test/foo/bar/baz.rs
     pub base: PathBuf,         // e.g., compile-test, auxiliary
     pub relative_dir: PathBuf, // e.g., foo/bar
 }

 #[derive(Debug)]
 pub struct TestDescAndFn {
     pub desc: TestDesc,
     pub testfn: TestFn,
 }

 #[derive(Default)]
 pub struct TestOpts {
     pub filter: Option<String>,
     pub run_ignored: bool,
     pub logfile: Option<PathBuf>,
     pub quiet: bool,
     pub test_threads: Option<usize>,
 }

 #[derive(Clone, PartialEq)]
 pub struct BenchSamples {
     ns_iter_summ: stats::Summary,
     mb_s: usize,
 }

 #[derive(Clone, PartialEq)]
 enum TestResult {
     TrIgnored,
     TrBench(BenchSamples),
 }

 unsafe impl Send for TestResult {}

 enum OutputLocation<T> {
     Raw(T),
 }

 struct ConsoleTestState<T> {
     log_out: Option<File>,
     out: OutputLocation<T>,
     quiet: bool,
     total: usize,
     passed: usize,
     failed: usize,
     ignored: usize,
     measured: usize,
     failures: Vec<(TestDesc, Vec<u8>)>,
     max_name_len: usize, // number of columns to fill when aligning names
 }

 impl ConsoleTestState<()> {
     pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState<io::Stdout>> {
         let log_out = match opts.logfile {
             Some(ref path) => Some(try!(File::create(path))),
             None => None,
         };
         let out = Raw(io::stdout());

         Ok(ConsoleTestState {
             out: out,
             log_out: log_out,
             quiet: opts.quiet,
             total: 0,
             passed: 0,
             failed: 0,
             ignored: 0,
             measured: 0,
             failures: Vec::new(),
             max_name_len: 0,
         })
     }
 }

 impl<T: Write> ConsoleTestState<T> {
     pub fn write_ignored(&mut self) -> io::Result<()> {
         self.write_short_result("ignored", "i")
     }

     pub fn write_bench(&mut self) -> io::Result<()> {
         self.write_pretty("bench")
     }

     pub fn write_short_result(&mut self, verbose: &str, quiet: &str)
                               -> io::Result<()> {
         if self.quiet {
             self.write_pretty(quiet)
         } else {
             try!(self.write_pretty(verbose));
             self.write_plain("\n")
         }
     }

     pub fn write_pretty(&mut self, word: &str) -> io::Result<()> {
         match self.out {
             Raw(ref mut stdout) => {
                 try!(stdout.write_all(word.as_bytes()));
                 stdout.flush()
             }
         }
     }

     pub fn write_plain(&mut self, s: &str) -> io::Result<()> {
         match self.out {
             Raw(ref mut stdout) => {
                 try!(stdout.write_all(s.as_bytes()));
                 stdout.flush()
             }
         }
     }

     pub fn write_run_start(&mut self, len: usize) -> io::Result<()> {
         self.total = len;
         let noun = if len != 1 {
             "tests"
         } else {
             "test"
         };
         self.write_plain(&format!("\nrunning {} {}\n", len, noun))
     }

     pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> {
         if self.quiet && align != PadOnRight {
             Ok(())
         } else {
             let name = test.padded_name(self.max_name_len, align);
             self.write_plain(&format!("test {} ... ", name))
         }
     }

     pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> {
         match *result {
             TrIgnored => self.write_ignored(),
             TrBench(ref bs) => {
                 try!(self.write_bench());
                 self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
             }
         }
     }

     pub fn write_log(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
         match self.log_out {
             None => Ok(()),
             Some(ref mut o) => {
                 let s = format!("{} {}\n",
                                 match *result {
                                     TrIgnored => "ignored".to_owned(),
                                     TrBench(ref bs) => fmt_bench_samples(bs),
                                 },
                                 test.name);
                 o.write_all(s.as_bytes())
             }
         }
     }

     pub fn write_failures(&mut self) -> io::Result<()> {
         try!(self.write_plain("\nfailures:\n"));
         let mut failures = Vec::new();
         let mut fail_out = String::new();
         for &(ref f, ref stdout) in &self.failures {
             failures.push(f.name.to_string());
             if !stdout.is_empty() {
                 fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
                 let output = String::from_utf8_lossy(stdout);
                 fail_out.push_str(&output);
                 fail_out.push_str("\n");
             }
         }
         if !fail_out.is_empty() {
             try!(self.write_plain("\n"));
             try!(self.write_plain(&fail_out));
         }

         try!(self.write_plain("\nfailures:\n"));
         failures.sort();
         for name in &failures {
             try!(self.write_plain(&format!("    {}\n", name)));
         }
         Ok(())
     }

     pub fn write_run_finish(&mut self) -> io::Result<bool> {
         assert_eq!(self.passed + self.failed + self.ignored + self.measured, self.total);

         let success = self.failed == 0;
         if !success {
             try!(self.write_failures());
         }

         try!(self.write_plain("\ntest result: "));
         if success {
             // There's no parallelism at this point so it's safe to use color
             try!(self.write_pretty("ok"));
         } else {
             try!(self.write_pretty("FAILED"));
         }
         let s = format!(". {} passed; {} failed; {} ignored; {} measured\n\n",
                         self.passed,
                         self.failed,
                         self.ignored,
                         self.measured);
         try!(self.write_plain(&s));
         Ok(success)
     }
 }

 // Format a number with thousands separators
 fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
     use std::fmt::Write;
     let mut output = String::new();
     let mut trailing = false;
     for &pow in &[9, 6, 3, 0] {
         let base = 10_usize.pow(pow);
         if pow == 0 || trailing || n / base != 0 {
             if !trailing {
                 output.write_fmt(format_args!("{}", n / base)).unwrap();
             } else {
                 output.write_fmt(format_args!("{:03}", n / base)).unwrap();
             }
             if pow != 0 {
                 output.push(sep);
             }
             trailing = true;
         }
         n %= base;
     }

     output
 }

 pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
     use std::fmt::Write;
     let mut output = String::new();

     let median = bs.ns_iter_summ.median as usize;
     let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;

     output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
                                   fmt_thousands_sep(median, ','),
                                   fmt_thousands_sep(deviation, ',')))
           .unwrap();
     if bs.mb_s != 0 {
         output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
     }
     output
 }

 // A simple console test runner
 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {

     fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> {
         match (*event).clone() {
             TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()),
             TeWait(ref test, padding) => st.write_test_start(test, padding),
             TeResult(test, result, _) => {
                 try!(st.write_log(&test, &result));
                 try!(st.write_result(&result));
                 match result {
                     TrIgnored => st.ignored += 1,
                     TrBench(_) => {
                         st.measured += 1
                     }
                 }
                 Ok(())
             }
         }
     }

     let mut st = try!(ConsoleTestState::new(opts));
     fn len_if_padded(t: &TestDescAndFn) -> usize {
         match t.testfn.padding() {
             PadOnRight => t.desc.name.len(),
         }
     }
     if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) {
         let n = &t.desc.name;
         st.max_name_len = n.len();
     }
     try!(run_tests(opts, tests, |x| callback(&x, &mut st)));
     st.write_run_finish()
 }

 #[test]
 fn should_sort_failures_before_printing_them() {
     let test_a = TestDesc {
         name: Cow::from("a"),
         ignore: false,
     };

     let test_b = TestDesc {
         name: Cow::from("b"),
         ignore: false,
     };

     let mut st = ConsoleTestState {
         log_out: None,
         out: Raw(Vec::new()),
         quiet: false,
         total: 0,
         passed: 0,
         failed: 0,
         ignored: 0,
         measured: 0,
         max_name_len: 10,
         failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
     };

     st.write_failures().unwrap();
     let s = match st.out {
         Raw(ref m) => String::from_utf8_lossy(&m[..]),
     };

     let apos = s.find("a").unwrap();
     let bpos = s.find("b").unwrap();
     assert!(apos < bpos);
 }

 #[derive(Clone)]
 enum TestEvent {
     TeFiltered(Vec<TestDesc>),
     TeWait(TestDesc, NamePadding),
     TeResult(TestDesc, TestResult, Vec<u8>),
 }

 type MonitorMsg = (TestDesc, TestResult, Vec<u8>);


 fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
     where F: FnMut(TestEvent) -> io::Result<()>
 {

     let filtered_tests = filter_tests(opts, tests);

     let filtered_descs = filtered_tests.iter()
                                        .map(|t| t.desc.clone())
                                        .collect();

     try!(callback(TeFiltered(filtered_descs)));

     let filtered_benchs_and_metrics = filtered_tests;

     // All benchmarks run at the end, in serial.
     // (this includes metric fns)
     for b in filtered_benchs_and_metrics {
         try!(callback(TeWait(b.desc.clone(), b.testfn.padding())));
         let (test, result, stdout) = run_test(opts, false, b);
         try!(callback(TeResult(test, result, stdout)));
     }
     Ok(())
 }

 fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
     let mut filtered = tests;

     // Remove tests that don't match the test filter
     filtered = match opts.filter {
         None => filtered,
         Some(ref filter) => {
             filtered.into_iter()
                     .filter(|test| test.desc.name.contains(&filter[..]))
                     .collect()
         }
     };

     // Maybe pull out the ignored test and unignore them
     filtered = if !opts.run_ignored {
         filtered
     } else {
         fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
             if test.desc.ignore {
                 let TestDescAndFn {desc, testfn} = test;
                 Some(TestDescAndFn {
                     desc: TestDesc { ignore: false, ..desc },
                     testfn: testfn,
                 })
             } else {
                 None
             }
         }
         filtered.into_iter().filter_map(filter).collect()
     };

     // Sort the tests alphabetically
     filtered.sort_by(|t1, t2| t1.desc.name.cmp(&t2.desc.name));

     filtered
 }

 fn run_test(_opts: &TestOpts,
             force_ignore: bool,
             test: TestDescAndFn) -> MonitorMsg
 {

     let TestDescAndFn {desc, testfn} = test;

     if force_ignore || desc.ignore {
         return (desc, TrIgnored, Vec::new());
     }

     match testfn {
         DynBenchFn(bencher) => {
             let bs = ::bench::benchmark(|harness| bencher.run(harness));
             (desc, TrBench(bs), Vec::new())
         }
         StaticBenchFn(benchfn) => {
             let bs = ::bench::benchmark(|harness| benchfn(harness));
             (desc, TrBench(bs), Vec::new())
         }
     }
 }


 // Benchmarking

 // FIXME: We don't have black_box in stable rust

 /// NOTE: We don't have a proper black box in stable Rust. This is
 /// a workaround implementation, that may have a too big performance overhead,
 /// depending on operation, or it may fail to properly avoid having code
 /// optimized out. It is good enough that it is used by default.
 ///
 /// A function that is opaque to the optimizer, to allow benchmarks to
 /// pretend to use outputs to assist in avoiding dead-code
 /// elimination.
 pub fn black_box<T>(dummy: T) -> T {
     unsafe {
         let ret = ptr::read_volatile(&dummy);
         forget(dummy);
         ret
     }
 }


 impl Bencher {
     /// Callback for benchmark functions to run in their body.
     pub fn iter<T, F>(&mut self, mut inner: F)
         where F: FnMut() -> T
     {
         let start = Instant::now();
         let k = self.iterations;
         for _ in 0..k {
             black_box(inner());
         }
         self.dur = start.elapsed();
     }

     pub fn ns_elapsed(&mut self) -> u64 {
         self.dur.as_secs() * 1_000_000_000 + (self.dur.subsec_nanos() as u64)
     }

     pub fn ns_per_iter(&mut self) -> u64 {
         if self.iterations == 0 {
             0
         } else {
             self.ns_elapsed() / cmp::max(self.iterations, 1)
         }
     }

     pub fn bench_n<F>(&mut self, n: u64, f: F)
         where F: FnOnce(&mut Bencher)
     {
         self.iterations = n;
         f(self);
     }

     // This is a more statistics-driven benchmark algorithm
     pub fn auto_bench<F>(&mut self, mut f: F) -> stats::Summary
         where F: FnMut(&mut Bencher)
     {
         // Initial bench run to get ballpark figure.
         let mut n = 1;
         self.bench_n(n, |x| f(x));

         // Try to estimate iter count for 1ms falling back to 1m
         // iterations if first run took < 1ns.
         if self.ns_per_iter() == 0 {
             n = 1_000_000;
         } else {
             n = 1_000_000 / cmp::max(self.ns_per_iter(), 1);
         }
         // if the first run took more than 1ms we don't want to just
         // be left doing 0 iterations on every loop. The unfortunate
         // side effect of not being able to do as many runs is
         // automatically handled by the statistical analysis below
         // (i.e. larger error bars).
         if n == 0 {
             n = 1;
         }

         let mut total_run = Duration::new(0, 0);
         let samples: &mut [f64] = &mut [0.0_f64; 50];
         loop {
             let loop_start = Instant::now();

             for p in &mut *samples {
                 self.bench_n(n, |x| f(x));
                 *p = self.ns_per_iter() as f64;
             }

             stats::winsorize(samples, 5.0);
             let summ = stats::Summary::new(samples);

             for p in &mut *samples {
                 self.bench_n(5 * n, |x| f(x));
                 *p = self.ns_per_iter() as f64;
             }

             stats::winsorize(samples, 5.0);
             let summ5 = stats::Summary::new(samples);
             let loop_run = loop_start.elapsed();

             // If we've run for 100ms and seem to have converged to a
             // stable median.
             if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
                summ.median - summ5.median < summ5.median_abs_dev {
                 return summ5;
             }

             total_run += loop_run;
             // Longest we ever run for is 3s.
             if total_run > Duration::from_secs(3) {
                 return summ5;
             }

             // If we overflow here just return the results so far. We check a
             // multiplier of 10 because we're about to multiply by 2 and the
             // next iteration of the loop will also multiply by 5 (to calculate
             // the summ5 result)
             n = match n.checked_mul(10) {
                 Some(_) => n * 2,
                 None => return summ5,
             };
         }
     }
 }

 pub mod bench {
     use std::cmp;
     use std::time::Duration;
     use super::{Bencher, BenchSamples};

     pub fn benchmark<F>(f: F) -> BenchSamples
         where F: FnMut(&mut Bencher)
     {
         let mut bs = Bencher {
             iterations: 0,
             dur: Duration::new(0, 0),
             bytes: 0,
         };

         let ns_iter_summ = bs.auto_bench(f);

         let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
         let mb_s = bs.bytes * 1000 / ns_iter;

         BenchSamples {
             ns_iter_summ: ns_iter_summ,
             mb_s: mb_s as usize,
         }
     }

     pub fn run_once<F>(f: F)
         where F: FnOnce(&mut Bencher)
     {
         let mut bs = Bencher {
             iterations: 0,
             dur: Duration::new(0, 0),
             bytes: 0,
         };
         bs.bench_n(1, f);
     }
 }
	// Copyright 2012-2016 The Rust Project Developers. See the COPYRIGHT
	// file at the top-level directory of this distribution and at
	// http://rust-lang.org/COPYRIGHT.
	//
	// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
	// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
	// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
	// option. This file may not be copied, modified, or distributed
	// except according to those terms.

	//! Simplified stable-compatible benchmark runner.
	//!
	//! Almost all user code will only be interested in `Bencher` and the
	//! macros that are used to describe benchmarker functions and
	//! the benchmark runner.
	//!
	//! NOTE: There's no proper `black_box` yet in this stable port of the
	//! benchmark runner, only a workaround implementation. It may not work
	//! exactly like the upstream `test::black_box`.
	//!
	//! One way to use this crate is to use it as dev-dependency and setup
	//! cargo to compile a file in `benches/` that runs without the testing harness.
	//!
	//! In Cargo.toml:
	//!
	//! ```ignore
	//! [[bench]]
	//! name = "example"
	//! harness = false
	//! ```
	//!
	//! In benches/example.rs:
	//!
	//! ```
	//! #[macro_use]
	//! extern crate bencher;
	//!
	//! use bencher::Bencher;
	//!
	//! fn a(bench: &mut Bencher) {
	//! bench.iter(\|\| {
	//! (0..1000).fold(0, \|x, y\| x + y)
	//! })
	//! }
	//!
	//! fn b(bench: &mut Bencher) {
	//! const N: usize = 1024;
	//! bench.iter(\|\| {
	//! vec![0u8; N]
	//! });
	//!
	//! bench.bytes = N as u64;
	//! }
	//!
	//! benchmark_group!(benches, a, b);
	//! benchmark_main!(benches);
	//!
	//! # #[cfg(never)]
	//! # fn main() { }
	//! ```
	//!
	//! Use `cargo bench` as usual. A command line argument can be used to filter
	//! which benchmarks to run.

	pub use self::TestFn::*;
	use self::TestResult::*;
	use self::TestEvent::*;
	use self::NamePadding::*;
	use self::OutputLocation::*;

	use std::borrow::Cow;
	use std::cmp;
	use std::fmt;
	use std::fs::File;
	use std::io::prelude::*;
	use std::io;
	use std::iter::repeat;
	use std::mem::forget;
	use std::path::PathBuf;
	use std::ptr;
	use std::time::{Instant, Duration};

	pub mod stats;
	mod macros;

	// The name of a test. By convention this follows the rules for rust
	// paths; i.e. it should be a series of identifiers separated by double
	// colons. This way if some test runner wants to arrange the tests
	// hierarchically it may.

	pub type TestName = Cow<'static, str>;

	#[derive(Clone, Copy, PartialEq, Eq)]
	enum NamePadding {
	PadOnRight,
	}

	impl TestDesc {
	fn padded_name(&self, column_count: usize, align: NamePadding) -> String {
	let mut name = self.name.to_string();
	let fill = column_count.saturating_sub(name.len());
	let pad = repeat(" ").take(fill).collect::<String>();
	match align {
	PadOnRight => {
	name.push_str(&pad);
	name
	}
	}
	}
	}

	/// Represents a benchmark function.
	pub trait TDynBenchFn: Send {
	fn run(&self, harness: &mut Bencher);
	}

	// A function that runs a test. If the function returns successfully,
	// the test succeeds; if the function panics then the test fails. We
	// may need to come up with a more clever definition of test in order
	// to support isolation of tests into threads.
	pub enum TestFn {
	StaticBenchFn(fn(&mut Bencher)),
	DynBenchFn(Box<TDynBenchFn + 'static>),
	}

	impl TestFn {
	fn padding(&self) -> NamePadding {
	match *self {
	StaticBenchFn(..) \|
	DynBenchFn(..) => PadOnRight,
	}
	}
	}

	impl fmt::Debug for TestFn {
	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
	f.write_str(match *self {
	StaticBenchFn(..) => "StaticBenchFn(..)",
	DynBenchFn(..) => "DynBenchFn(..)",
	})
	}
	}

	/// Manager of the benchmarking runs.
	///
	/// This is fed into functions marked with `#[bench]` to allow for
	/// set-up & tear-down before running a piece of code repeatedly via a
	/// call to `iter`.
	#[derive(Copy, Clone)]
	pub struct Bencher {
	iterations: u64,
	dur: Duration,
	pub bytes: u64,
	}

	// The definition of a single test. A test runner will run a list of
	// these.
	#[derive(Clone, Debug, PartialEq, Eq, Hash)]
	pub struct TestDesc {
	pub name: TestName,
	pub ignore: bool,
	}

	#[derive(Clone)]
	pub struct TestPaths {
	pub file: PathBuf, // e.g., compile-test/foo/bar/baz.rs
	pub base: PathBuf, // e.g., compile-test, auxiliary
	pub relative_dir: PathBuf, // e.g., foo/bar
	}

	#[derive(Debug)]
	pub struct TestDescAndFn {
	pub desc: TestDesc,
	pub testfn: TestFn,
	}

	#[derive(Default)]
	pub struct TestOpts {
	pub filter: Option<String>,
	pub run_ignored: bool,
	pub logfile: Option<PathBuf>,
	pub quiet: bool,
	pub test_threads: Option<usize>,
	}

	#[derive(Clone, PartialEq)]
	pub struct BenchSamples {
	ns_iter_summ: stats::Summary,
	mb_s: usize,
	}

	#[derive(Clone, PartialEq)]
	enum TestResult {
	TrIgnored,
	TrBench(BenchSamples),
	}

	unsafe impl Send for TestResult {}

	enum OutputLocation<T> {
	Raw(T),
	}

	struct ConsoleTestState<T> {
	log_out: Option<File>,
	out: OutputLocation<T>,
	quiet: bool,
	total: usize,
	passed: usize,
	failed: usize,
	ignored: usize,
	measured: usize,
	failures: Vec<(TestDesc, Vec<u8>)>,
	max_name_len: usize, // number of columns to fill when aligning names
	}

	impl ConsoleTestState<()> {
	pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState<io::Stdout>> {
	let log_out = match opts.logfile {
	Some(ref path) => Some(try!(File::create(path))),
	None => None,
	};
	let out = Raw(io::stdout());

	Ok(ConsoleTestState {
	out: out,
	log_out: log_out,
	quiet: opts.quiet,
	total: 0,
	passed: 0,
	failed: 0,
	ignored: 0,
	measured: 0,
	failures: Vec::new(),
	max_name_len: 0,
	})
	}
	}

	impl<T: Write> ConsoleTestState<T> {
	pub fn write_ignored(&mut self) -> io::Result<()> {
	self.write_short_result("ignored", "i")
	}

	pub fn write_bench(&mut self) -> io::Result<()> {
	self.write_pretty("bench")
	}

	pub fn write_short_result(&mut self, verbose: &str, quiet: &str)
	-> io::Result<()> {
	if self.quiet {
	self.write_pretty(quiet)
	} else {
	try!(self.write_pretty(verbose));
	self.write_plain("\n")
	}
	}

	pub fn write_pretty(&mut self, word: &str) -> io::Result<()> {
	match self.out {
	Raw(ref mut stdout) => {
	try!(stdout.write_all(word.as_bytes()));
	stdout.flush()
	}
	}
	}

	pub fn write_plain(&mut self, s: &str) -> io::Result<()> {
	match self.out {
	Raw(ref mut stdout) => {
	try!(stdout.write_all(s.as_bytes()));
	stdout.flush()
	}
	}
	}

	pub fn write_run_start(&mut self, len: usize) -> io::Result<()> {
	self.total = len;
	let noun = if len != 1 {
	"tests"
	} else {
	"test"
	};
	self.write_plain(&format!("\nrunning {} {}\n", len, noun))
	}

	pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> {
	if self.quiet && align != PadOnRight {
	Ok(())
	} else {
	let name = test.padded_name(self.max_name_len, align);
	self.write_plain(&format!("test {} ... ", name))
	}
	}

	pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> {
	match *result {
	TrIgnored => self.write_ignored(),
	TrBench(ref bs) => {
	try!(self.write_bench());
	self.write_plain(&format!(": {}\n", fmt_bench_samples(bs)))
	}
	}
	}

	pub fn write_log(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> {
	match self.log_out {
	None => Ok(()),
	Some(ref mut o) => {
	let s = format!("{} {}\n",
	match *result {
	TrIgnored => "ignored".to_owned(),
	TrBench(ref bs) => fmt_bench_samples(bs),
	},
	test.name);
	o.write_all(s.as_bytes())
	}
	}
	}

	pub fn write_failures(&mut self) -> io::Result<()> {
	try!(self.write_plain("\nfailures:\n"));
	let mut failures = Vec::new();
	let mut fail_out = String::new();
	for &(ref f, ref stdout) in &self.failures {
	failures.push(f.name.to_string());
	if !stdout.is_empty() {
	fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name));
	let output = String::from_utf8_lossy(stdout);
	fail_out.push_str(&output);
	fail_out.push_str("\n");
	}
	}
	if !fail_out.is_empty() {
	try!(self.write_plain("\n"));
	try!(self.write_plain(&fail_out));
	}

	try!(self.write_plain("\nfailures:\n"));
	failures.sort();
	for name in &failures {
	try!(self.write_plain(&format!(" {}\n", name)));
	}
	Ok(())
	}

	pub fn write_run_finish(&mut self) -> io::Result<bool> {
	assert_eq!(self.passed + self.failed + self.ignored + self.measured, self.total);

	let success = self.failed == 0;
	if !success {
	try!(self.write_failures());
	}

	try!(self.write_plain("\ntest result: "));
	if success {
	// There's no parallelism at this point so it's safe to use color
	try!(self.write_pretty("ok"));
	} else {
	try!(self.write_pretty("FAILED"));
	}
	let s = format!(". {} passed; {} failed; {} ignored; {} measured\n\n",
	self.passed,
	self.failed,
	self.ignored,
	self.measured);
	try!(self.write_plain(&s));
	Ok(success)
	}
	}

	// Format a number with thousands separators
	fn fmt_thousands_sep(mut n: usize, sep: char) -> String {
	use std::fmt::Write;
	let mut output = String::new();
	let mut trailing = false;
	for &pow in &[9, 6, 3, 0] {
	let base = 10_usize.pow(pow);
	if pow == 0 \|\| trailing \|\| n / base != 0 {
	if !trailing {
	output.write_fmt(format_args!("{}", n / base)).unwrap();
	} else {
	output.write_fmt(format_args!("{:03}", n / base)).unwrap();
	}
	if pow != 0 {
	output.push(sep);
	}
	trailing = true;
	}
	n %= base;
	}

	output
	}

	pub fn fmt_bench_samples(bs: &BenchSamples) -> String {
	use std::fmt::Write;
	let mut output = String::new();

	let median = bs.ns_iter_summ.median as usize;
	let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize;

	output.write_fmt(format_args!("{:>11} ns/iter (+/- {})",
	fmt_thousands_sep(median, ','),
	fmt_thousands_sep(deviation, ',')))
	.unwrap();
	if bs.mb_s != 0 {
	output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap();
	}
	output
	}

	// A simple console test runner
	pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {

	fn callback<T: Write>(event: &TestEvent, st: &mut ConsoleTestState<T>) -> io::Result<()> {
	match (*event).clone() {
	TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()),
	TeWait(ref test, padding) => st.write_test_start(test, padding),
	TeResult(test, result, _) => {
	try!(st.write_log(&test, &result));
	try!(st.write_result(&result));
	match result {
	TrIgnored => st.ignored += 1,
	TrBench(_) => {
	st.measured += 1
	}
	}
	Ok(())
	}
	}
	}

	let mut st = try!(ConsoleTestState::new(opts));
	fn len_if_padded(t: &TestDescAndFn) -> usize {
	match t.testfn.padding() {
	PadOnRight => t.desc.name.len(),
	}
	}
	if let Some(t) = tests.iter().max_by_key(\|t\| len_if_padded(*t)) {
	let n = &t.desc.name;
	st.max_name_len = n.len();
	}
	try!(run_tests(opts, tests, \|x\| callback(&x, &mut st)));
	st.write_run_finish()
	}

	#[test]
	fn should_sort_failures_before_printing_them() {
	let test_a = TestDesc {
	name: Cow::from("a"),
	ignore: false,
	};

	let test_b = TestDesc {
	name: Cow::from("b"),
	ignore: false,
	};

	let mut st = ConsoleTestState {
	log_out: None,
	out: Raw(Vec::new()),
	quiet: false,
	total: 0,
	passed: 0,
	failed: 0,
	ignored: 0,
	measured: 0,
	max_name_len: 10,
	failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
	};

	st.write_failures().unwrap();
	let s = match st.out {
	Raw(ref m) => String::from_utf8_lossy(&m[..]),
	};

	let apos = s.find("a").unwrap();
	let bpos = s.find("b").unwrap();
	assert!(apos < bpos);
	}

	#[derive(Clone)]
	enum TestEvent {
	TeFiltered(Vec<TestDesc>),
	TeWait(TestDesc, NamePadding),
	TeResult(TestDesc, TestResult, Vec<u8>),
	}

	type MonitorMsg = (TestDesc, TestResult, Vec<u8>);


	fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
	where F: FnMut(TestEvent) -> io::Result<()>
	{

	let filtered_tests = filter_tests(opts, tests);

	let filtered_descs = filtered_tests.iter()
	.map(\|t\| t.desc.clone())
	.collect();

	try!(callback(TeFiltered(filtered_descs)));

	let filtered_benchs_and_metrics = filtered_tests;

	// All benchmarks run at the end, in serial.
	// (this includes metric fns)
	for b in filtered_benchs_and_metrics {
	try!(callback(TeWait(b.desc.clone(), b.testfn.padding())));
	let (test, result, stdout) = run_test(opts, false, b);
	try!(callback(TeResult(test, result, stdout)));
	}
	Ok(())
	}

	fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
	let mut filtered = tests;

	// Remove tests that don't match the test filter
	filtered = match opts.filter {
	None => filtered,
	Some(ref filter) => {
	filtered.into_iter()
	.filter(\|test\| test.desc.name.contains(&filter[..]))
	.collect()
	}
	};

	// Maybe pull out the ignored test and unignore them
	filtered = if !opts.run_ignored {
	filtered
	} else {
	fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> {
	if test.desc.ignore {
	let TestDescAndFn {desc, testfn} = test;
	Some(TestDescAndFn {
	desc: TestDesc { ignore: false, ..desc },
	testfn: testfn,
	})
	} else {
	None
	}
	}
	filtered.into_iter().filter_map(filter).collect()
	};

	// Sort the tests alphabetically
	filtered.sort_by(\|t1, t2\| t1.desc.name.cmp(&t2.desc.name));

	filtered
	}

	fn run_test(_opts: &TestOpts,
	force_ignore: bool,
	test: TestDescAndFn) -> MonitorMsg
	{

	let TestDescAndFn {desc, testfn} = test;

	if force_ignore \|\| desc.ignore {
	return (desc, TrIgnored, Vec::new());
	}

	match testfn {
	DynBenchFn(bencher) => {
	let bs = ::bench::benchmark(\|harness\| bencher.run(harness));
	(desc, TrBench(bs), Vec::new())
	}
	StaticBenchFn(benchfn) => {
	let bs = ::bench::benchmark(\|harness\| benchfn(harness));
	(desc, TrBench(bs), Vec::new())
	}
	}
	}


	// Benchmarking

	// FIXME: We don't have black_box in stable rust

	/// NOTE: We don't have a proper black box in stable Rust. This is
	/// a workaround implementation, that may have a too big performance overhead,
	/// depending on operation, or it may fail to properly avoid having code
	/// optimized out. It is good enough that it is used by default.
	///
	/// A function that is opaque to the optimizer, to allow benchmarks to
	/// pretend to use outputs to assist in avoiding dead-code
	/// elimination.
	pub fn black_box<T>(dummy: T) -> T {
	unsafe {
	let ret = ptr::read_volatile(&dummy);
	forget(dummy);
	ret
	}
	}


	impl Bencher {
	/// Callback for benchmark functions to run in their body.
	pub fn iter<T, F>(&mut self, mut inner: F)
	where F: FnMut() -> T
	{
	let start = Instant::now();
	let k = self.iterations;
	for _ in 0..k {
	black_box(inner());
	}
	self.dur = start.elapsed();
	}

	pub fn ns_elapsed(&mut self) -> u64 {
	self.dur.as_secs() * 1_000_000_000 + (self.dur.subsec_nanos() as u64)
	}

	pub fn ns_per_iter(&mut self) -> u64 {
	if self.iterations == 0 {
	0
	} else {
	self.ns_elapsed() / cmp::max(self.iterations, 1)
	}
	}

	pub fn bench_n<F>(&mut self, n: u64, f: F)
	where F: FnOnce(&mut Bencher)
	{
	self.iterations = n;
	f(self);
	}

	// This is a more statistics-driven benchmark algorithm
	pub fn auto_bench<F>(&mut self, mut f: F) -> stats::Summary
	where F: FnMut(&mut Bencher)
	{
	// Initial bench run to get ballpark figure.
	let mut n = 1;
	self.bench_n(n, \|x\| f(x));

	// Try to estimate iter count for 1ms falling back to 1m
	// iterations if first run took < 1ns.
	if self.ns_per_iter() == 0 {
	n = 1_000_000;
	} else {
	n = 1_000_000 / cmp::max(self.ns_per_iter(), 1);
	}
	// if the first run took more than 1ms we don't want to just
	// be left doing 0 iterations on every loop. The unfortunate
	// side effect of not being able to do as many runs is
	// automatically handled by the statistical analysis below
	// (i.e. larger error bars).
	if n == 0 {
	n = 1;
	}

	let mut total_run = Duration::new(0, 0);
	let samples: &mut [f64] = &mut [0.0_f64; 50];
	loop {
	let loop_start = Instant::now();

	for p in &mut *samples {
	self.bench_n(n, \|x\| f(x));
	*p = self.ns_per_iter() as f64;
	}

	stats::winsorize(samples, 5.0);
	let summ = stats::Summary::new(samples);

	for p in &mut *samples {
	self.bench_n(5 * n, \|x\| f(x));
	*p = self.ns_per_iter() as f64;
	}

	stats::winsorize(samples, 5.0);
	let summ5 = stats::Summary::new(samples);
	let loop_run = loop_start.elapsed();

	// If we've run for 100ms and seem to have converged to a
	// stable median.
	if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 &&
	summ.median - summ5.median < summ5.median_abs_dev {
	return summ5;
	}

	total_run += loop_run;
	// Longest we ever run for is 3s.
	if total_run > Duration::from_secs(3) {
	return summ5;
	}

	// If we overflow here just return the results so far. We check a
	// multiplier of 10 because we're about to multiply by 2 and the
	// next iteration of the loop will also multiply by 5 (to calculate
	// the summ5 result)
	n = match n.checked_mul(10) {
	Some(_) => n * 2,
	None => return summ5,
	};
	}
	}
	}

	pub mod bench {
	use std::cmp;
	use std::time::Duration;
	use super::{Bencher, BenchSamples};

	pub fn benchmark<F>(f: F) -> BenchSamples
	where F: FnMut(&mut Bencher)
	{
	let mut bs = Bencher {
	iterations: 0,
	dur: Duration::new(0, 0),
	bytes: 0,
	};

	let ns_iter_summ = bs.auto_bench(f);

	let ns_iter = cmp::max(ns_iter_summ.median as u64, 1);
	let mb_s = bs.bytes * 1000 / ns_iter;

	BenchSamples {
	ns_iter_summ: ns_iter_summ,
	mb_s: mb_s as usize,
	}
	}

	pub fn run_once<F>(f: F)
	where F: FnOnce(&mut Bencher)
	{
	let mut bs = Bencher {
	iterations: 0,
	dur: Duration::new(0, 0),
	bytes: 0,
	};
	bs.bench_n(1, f);
	}
	}