Snap for 6439596 from 20f5b67313dbf0ddafb792a5a2cc298b9cc58063 to qt-aml-tzdata-release
Change-Id: Icfa12102074483c104af8f55acad903495be6e3f
diff --git a/.clang-format b/.clang-format
index e7d00fe..4b3f13f 100644
--- a/.clang-format
+++ b/.clang-format
@@ -1,5 +1,5 @@
---
Language: Cpp
BasedOnStyle: Google
-PointerAlignment: Left
...
+
diff --git a/.gitignore b/.gitignore
index a7716e3..8c30e28 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,7 +8,6 @@
!/cmake/*.cmake
!/test/AssemblyTests.cmake
*~
-*.swp
*.pyc
__pycache__
@@ -57,6 +56,3 @@
# Visual Studio 2015/2017 cache/options directory
.vs/
CMakeSettings.json
-
-# Visual Studio Code cache/options directory
-.vscode/
diff --git a/.travis.yml b/.travis.yml
index 6b6cfc7..4625dfb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,25 +23,13 @@
apt:
packages:
- g++-multilib
- - libc6:i386
- env:
- - COMPILER=g++
- - C_COMPILER=gcc
- - BUILD_TYPE=Debug
- - BUILD_32_BITS=ON
- - EXTRA_FLAGS="-m32"
+ env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug BUILD_32_BITS=ON
- compiler: gcc
addons:
apt:
packages:
- g++-multilib
- - libc6:i386
- env:
- - COMPILER=g++
- - C_COMPILER=gcc
- - BUILD_TYPE=Release
- - BUILD_32_BITS=ON
- - EXTRA_FLAGS="-m32"
+ env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release BUILD_32_BITS=ON
- compiler: gcc
env:
- INSTALL_GCC6_FROM_PPA=1
@@ -54,102 +42,81 @@
env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Release
# Clang w/ libc++
- compiler: clang
- dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- - INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug
- LIBCXX_BUILD=1
- - EXTRA_CXX_FLAGS="-stdlib=libc++"
+ - EXTRA_FLAGS="-stdlib=libc++"
- compiler: clang
- dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- - INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release
- LIBCXX_BUILD=1
- - EXTRA_CXX_FLAGS="-stdlib=libc++"
+ - EXTRA_FLAGS="-stdlib=libc++"
# Clang w/ 32bit libc++
- compiler: clang
- dist: xenial
addons:
apt:
packages:
- clang-3.8
- g++-multilib
- - libc6:i386
env:
- - INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug
- LIBCXX_BUILD=1
- BUILD_32_BITS=ON
- - EXTRA_FLAGS="-m32"
- - EXTRA_CXX_FLAGS="-stdlib=libc++"
+ - EXTRA_FLAGS="-stdlib=libc++ -m32"
# Clang w/ 32bit libc++
- compiler: clang
- dist: xenial
addons:
apt:
packages:
- clang-3.8
- g++-multilib
- - libc6:i386
env:
- - INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release
- LIBCXX_BUILD=1
- BUILD_32_BITS=ON
- - EXTRA_FLAGS="-m32"
- - EXTRA_CXX_FLAGS="-stdlib=libc++"
+ - EXTRA_FLAGS="-stdlib=libc++ -m32"
# Clang w/ libc++, ASAN, UBSAN
- compiler: clang
- dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- - INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug
- LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address"
- ENABLE_SANITIZER=1
- - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all"
- - EXTRA_CXX_FLAGS="-stdlib=libc++"
+ - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all"
- UBSAN_OPTIONS=print_stacktrace=1
# Clang w/ libc++ and MSAN
- compiler: clang
- dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- - INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug
- LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins
- ENABLE_SANITIZER=1
- - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins"
- - EXTRA_CXX_FLAGS="-stdlib=libc++"
+ - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins"
# Clang w/ libc++ and MSAN
- compiler: clang
- dist: xenial
addons:
apt:
packages:
clang-3.8
env:
- - INSTALL_GCC6_FROM_PPA=1
- COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo
- LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread
- ENABLE_SANITIZER=1
- - EXTRA_FLAGS="-g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all"
- - EXTRA_CXX_FLAGS="-stdlib=libc++"
+ - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all"
- os: osx
osx_image: xcode8.3
compiler: clang
@@ -164,10 +131,7 @@
osx_image: xcode8.3
compiler: clang
env:
- - COMPILER=clang++
- - BUILD_TYPE=Release
- - BUILD_32_BITS=ON
- - EXTRA_FLAGS="-m32"
+ - COMPILER=clang++ BUILD_TYPE=Release BUILD_32_BITS=ON
- os: osx
osx_image: xcode8.3
compiler: gcc
@@ -214,7 +178,7 @@
fi
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
sudo apt-get update -qq;
- sudo apt-get install -qq unzip cmake3;
+ sudo apt-get install -qq unzip;
wget https://github.com/bazelbuild/bazel/releases/download/0.10.1/bazel-0.10.1-installer-linux-x86_64.sh --output-document bazel-installer.sh;
travis_wait sudo bash bazel-installer.sh;
fi
@@ -224,7 +188,7 @@
fi
script:
- - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_C_FLAGS="${EXTRA_FLAGS}" -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS} ${EXTRA_CXX_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} ..
+ - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} ${EXTRA_OPTIONS} ..
- make
- ctest -C ${BUILD_TYPE} --output-on-failure
- bazel test -c dbg --define google_benchmark.have_regex=posix --announce_rc --verbose_failures --test_output=errors --keep_going //test/...
diff --git a/AUTHORS b/AUTHORS
index 35c4c8c..ef3051a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -9,18 +9,14 @@
# Please keep the list sorted.
Albert Pretorius <pretoalb@gmail.com>
-Alex Steele <steeleal123@gmail.com>
Andriy Berestovskyy <berestovskyy@gmail.com>
Arne Beer <arne@twobeer.de>
Carto
Christopher Seymour <chris.j.seymour@hotmail.com>
-Colin Braley <braley.colin@gmail.com>
-Daniel Harvey <danielharvey458@gmail.com>
David Coeurjolly <david.coeurjolly@liris.cnrs.fr>
Deniz Evrenci <denizevrenci@gmail.com>
Dirac Research
Dominik Czarnota <dominik.b.czarnota@gmail.com>
-Eric Backus <eric_backus@alum.mit.edu>
Eric Fiselier <eric@efcs.ca>
Eugene Zhuk <eugene.zhuk@gmail.com>
Evgeny Safronov <division494@gmail.com>
@@ -45,7 +41,6 @@
Paul Redmond <paul.redmond@gmail.com>
Radoslav Yovchev <radoslav.tm@gmail.com>
Roman Lebedev <lebedev.ri@gmail.com>
-Sayan Bhattacharjee <aero.sayan@gmail.com>
Shuo Chen <chenshuo@chenshuo.com>
Steinar H. Gunderson <sgunderson@bigfoot.com>
Stripe, Inc.
diff --git a/Android.bp b/Android.bp
index e69336d..0ec7d5e 100644
--- a/Android.bp
+++ b/Android.bp
@@ -18,6 +18,7 @@
name: "libgoogle-benchmark",
host_supported: true,
local_include_dirs: ["include"],
+ vendor_available: true,
cflags: [
"-DBENCHMARK_ANDROID",
"-DHAVE_POSIX_REGEX",
@@ -25,11 +26,24 @@
"-Wno-deprecated-declarations",
],
- exclude_srcs: [
- "src/benchmark_main.cc",
- ],
srcs: [
- "src/*.cc",
+ "src/benchmark_api_internal.cc",
+ "src/benchmark.cc",
+ "src/benchmark_register.cc",
+ "src/benchmark_runner.cc",
+ "src/colorprint.cc",
+ "src/commandlineflags.cc",
+ "src/complexity.cc",
+ "src/console_reporter.cc",
+ "src/counter.cc",
+ "src/csv_reporter.cc",
+ "src/json_reporter.cc",
+ "src/reporter.cc",
+ "src/sleep.cc",
+ "src/statistics.cc",
+ "src/string_util.cc",
+ "src/sysinfo.cc",
+ "src/timers.cc",
],
export_include_dirs: ["include"],
}
diff --git a/BUILD.bazel b/BUILD.bazel
index d97a019..6ee69f2 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -8,8 +8,6 @@
visibility = [":__subpackages__"],
)
-load("@rules_cc//cc:defs.bzl", "cc_library")
-
cc_library(
name = "benchmark",
srcs = glob(
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8cfe125..d7ed57e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,18 +1,17 @@
-cmake_minimum_required (VERSION 3.5.1)
+cmake_minimum_required (VERSION 2.8.12)
foreach(p
CMP0048 # OK to clear PROJECT_VERSION on project()
CMP0054 # CMake 3.1
CMP0056 # export EXE_LINKER_FLAGS to try_run
CMP0057 # Support no if() IN_LIST operator
- CMP0063 # Honor visibility properties for all targets
)
if(POLICY ${p})
cmake_policy(SET ${p} NEW)
endif()
endforeach()
-project (benchmark CXX)
+project (benchmark)
option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON)
option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON)
@@ -33,7 +32,6 @@
# in cases where it is not possible to build or find a valid version of gtest.
option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" ON)
-set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
set(ENABLE_ASSEMBLY_TESTS_DEFAULT OFF)
function(should_enable_assembly_tests)
if(CMAKE_BUILD_TYPE)
@@ -192,7 +190,7 @@
# Link time optimisation
if (BENCHMARK_ENABLE_LTO)
add_cxx_compiler_flag(-flto)
- if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+ if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
find_program(GCC_AR gcc-ar)
if (GCC_AR)
set(CMAKE_AR ${GCC_AR})
@@ -201,7 +199,7 @@
if (GCC_RANLIB)
set(CMAKE_RANLIB ${GCC_RANLIB})
endif()
- elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+ elseif("${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
include(llvm-toolchain)
endif()
endif()
@@ -268,10 +266,8 @@
if (BENCHMARK_ENABLE_TESTING)
enable_testing()
- if (BENCHMARK_ENABLE_GTEST_TESTS AND
- NOT (TARGET gtest AND TARGET gtest_main AND
- TARGET gmock AND TARGET gmock_main))
- include(GoogleTest)
+ if (BENCHMARK_ENABLE_GTEST_TESTS)
+ include(HandleGTest)
endif()
add_subdirectory(test)
endif()
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 6b64a00..d0c31df 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -23,27 +23,21 @@
# Please keep the list sorted.
Albert Pretorius <pretoalb@gmail.com>
-Alex Steele <steelal123@gmail.com>
Andriy Berestovskyy <berestovskyy@gmail.com>
Arne Beer <arne@twobeer.de>
Billy Robert O'Neal III <billy.oneal@gmail.com> <bion@microsoft.com>
Chris Kennelly <ckennelly@google.com> <ckennelly@ckennelly.com>
Christopher Seymour <chris.j.seymour@hotmail.com>
-Colin Braley <braley.colin@gmail.com>
Cyrille Faucheux <cyrille.faucheux@gmail.com>
-Daniel Harvey <danielharvey458@gmail.com>
David Coeurjolly <david.coeurjolly@liris.cnrs.fr>
Deniz Evrenci <denizevrenci@gmail.com>
Dominic Hamon <dma@stripysock.com> <dominic@google.com>
Dominik Czarnota <dominik.b.czarnota@gmail.com>
-Eric Backus <eric_backus@alum.mit.edu>
Eric Fiselier <eric@efcs.ca>
Eugene Zhuk <eugene.zhuk@gmail.com>
Evgeny Safronov <division494@gmail.com>
Federico Ficarelli <federico.ficarelli@gmail.com>
Felix Homann <linuxaudio@showlabor.de>
-Geoffrey Martin-Noble <gcmn@google.com> <gmngeoffrey@gmail.com>
-Hannes Hauswedell <h2@fsfe.org>
Ismael Jimenez Martinez <ismael.jimenez.martinez@gmail.com>
Jern-Kuan Leong <jernkuan@gmail.com>
JianXiong Zhou <zhoujianxiong2@gmail.com>
@@ -51,8 +45,8 @@
John Millikin <jmillikin@stripe.com>
Jussi Knuuttila <jussi.knuuttila@gmail.com>
Kai Wolf <kai.wolf@gmail.com>
-Kaito Udagawa <umireon@gmail.com>
Kishan Kumar <kumar.kishan@outlook.com>
+Kaito Udagawa <umireon@gmail.com>
Lei Xu <eddyxu@gmail.com>
Matt Clarkson <mattyclarkson@gmail.com>
Maxim Vafin <maxvafin@gmail.com>
@@ -67,7 +61,6 @@
Ray Glover <ray.glover@uk.ibm.com>
Robert Guo <robert.guo@mongodb.com>
Roman Lebedev <lebedev.ri@gmail.com>
-Sayan Bhattacharjee <aero.sayan@gmail.com>
Shuo Chen <chenshuo@chenshuo.com>
Tobias Ulvgård <tobias.ulvgard@dirac.se>
Tom Madams <tom.ej.madams@gmail.com> <tmadams@google.com>
diff --git a/METADATA b/METADATA
index fb2362a..b0ac15d 100644
--- a/METADATA
+++ b/METADATA
@@ -9,10 +9,10 @@
type: GIT
value: "https://github.com/google/benchmark.git"
}
- version: "c50ac68c50ff8da3827cd6720792117910d85666"
+ version: "d205ead299c7cddd5e1bc3478d57ad4320a4a53c"
last_upgrade_date {
year: 2019
- month: 11
- day: 6
+ month: 3
+ day: 4
}
}
diff --git a/README.md b/README.md
index a8aa276..902915e 100644
--- a/README.md
+++ b/README.md
@@ -1,36 +1,10 @@
-# Benchmark
+# benchmark
[](https://travis-ci.org/google/benchmark)
[](https://ci.appveyor.com/project/google/benchmark/branch/master)
[](https://coveralls.io/r/google/benchmark)
[](https://slackin-iqtfqnpzxd.now.sh/)
-
-A library to benchmark code snippets, similar to unit tests. Example:
-
-```c++
-#include <benchmark/benchmark.h>
-
-static void BM_SomeFunction(benchmark::State& state) {
- // Perform setup here
- for (auto _ : state) {
- // This code gets timed
- SomeFunction();
- }
-}
-// Register the function as a benchmark
-BENCHMARK(BM_SomeFunction);
-// Run the benchmark
-BENCHMARK_MAIN();
-```
-
-To get started, see [Requirements](#requirements) and
-[Installation](#installation). See [Usage](#usage) for a full example and the
-[User Guide](#user-guide) for a more comprehensive feature overview.
-
-It may also help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/googletest/docs/primer.md)
-as some of the structural aspects of the APIs are similar.
-
-### Resources
+A library to support the benchmarking of functions, similar to unit-tests.
[Discussion group](https://groups.google.com/d/forum/benchmark-discuss)
@@ -40,68 +14,20 @@
[Assembly Testing Documentation](docs/AssemblyTests.md)
-## Requirements
-The library can be used with C++03. However, it requires C++11 to build,
-including compiler and standard library support.
+## Building
-The following minimum versions are required to build the library:
-
-* GCC 4.8
-* Clang 3.4
-* Visual Studio 14 2015
-* Intel 2015 Update 1
-
-See [Platform-Specific Build Instructions](#platform-specific-build-instructions).
-
-## Installation
-
-This describes the installation process using cmake. As pre-requisites, you'll
-need git and cmake installed.
-
-_See [dependencies.md](dependencies.md) for more details regarding supported
-versions of build tools._
+The basic steps for configuring and building the library look like this:
```bash
-# Check out the library.
$ git clone https://github.com/google/benchmark.git
# Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory.
$ git clone https://github.com/google/googletest.git benchmark/googletest
-# Go to the library root directory
-$ cd benchmark
-# Make a build directory to place the build output.
$ mkdir build && cd build
-# Generate a Makefile with cmake.
-# Use cmake -G <generator> to generate a different file type.
-$ cmake ../
-# Build the library.
-# Use make -j<number_of_parallel_jobs> to speed up the build process, e.g. make -j8 .
+$ cmake -G <generator> [options] ../benchmark
+# Assuming a makefile generator was used
$ make
```
-This builds the `benchmark` and `benchmark_main` libraries and tests.
-On a unix system, the build directory should now look something like this:
-
-```
-/benchmark
- /build
- /src
- /libbenchmark.a
- /libbenchmark_main.a
- /test
- ...
-```
-
-Next, you can run the tests to check the build.
-
-```bash
-$ make test
-```
-
-If you want to install the library globally, also run:
-
-```
-sudo make install
-```
Note that Google Benchmark requires Google Test to build and run the tests. This
dependency can be provided two ways:
@@ -114,29 +40,37 @@
If you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF`
to `CMAKE_ARGS`.
-### Debug vs Release
-By default, benchmark builds as a debug library. You will see a warning in the
-output when this is the case. To build it as a release library instead, use:
+## Installation Guide
+
+For Ubuntu and Debian Based System
+
+First make sure you have git and cmake installed (If not please install them)
```
-cmake -DCMAKE_BUILD_TYPE=Release
+sudo apt-get install git cmake
```
-To enable link-time optimisation, use
+Now, let's clone the repository and build it
```
-cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true
+git clone https://github.com/google/benchmark.git
+cd benchmark
+# If you want to build tests and don't use BENCHMARK_DOWNLOAD_DEPENDENCIES, then
+# git clone https://github.com/google/googletest.git
+mkdir build
+cd build
+cmake .. -DCMAKE_BUILD_TYPE=RELEASE
+make
```
-If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake
-cache variables, if autodetection fails.
+If you need to install the library globally
-If you are using clang, you may need to set `LLVMAR_EXECUTABLE`,
-`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables.
+```
+sudo make install
+```
-
-### Stable and Experimental Library Versions
+## Stable and Experimental Library Versions
The main branch contains the latest stable version of the benchmarking library;
the API of which can be considered largely stable, with source breaking changes
@@ -148,11 +82,16 @@
this branch. However, this branch provides no stability guarantees and reserves
the right to change and break the API at any time.
-## Usage
+## Further knowledge
+
+It may help to read the [Google Test documentation](https://github.com/google/googletest/blob/master/googletest/docs/primer.md)
+as some of the structural aspects of the APIs are similar.
+
+## Example usage
### Basic usage
-Define a function that executes the code to measure, register it as a benchmark
-function using the `BENCHMARK` macro, and ensure an appropriate `main` function
-is available:
+Define a function that executes the code to be measured, register it as a
+benchmark function using the `BENCHMARK` macro, and ensure an appropriate `main`
+function is available:
```c++
#include <benchmark/benchmark.h>
@@ -175,27 +114,15 @@
BENCHMARK_MAIN();
```
-To run the benchmark, compile and link against the `benchmark` library
-(libbenchmark.a/.so). If you followed the build steps above, this
-library will be under the build directory you created.
+Don't forget to inform your linker to add benchmark library e.g. through
+`-lbenchmark` compilation flag. Alternatively, you may leave out the
+`BENCHMARK_MAIN();` at the end of the source file and link against
+`-lbenchmark_main` to get the same default behavior.
-```bash
-# Example on linux after running the build steps above. Assumes the
-# `benchmark` and `build` directories are under the current directory.
-$ g++ mybenchmark.cc -std=c++11 -isystem benchmark/include \
- -Lbenchmark/build/src -lbenchmark -lpthread -o mybenchmark
-```
+The benchmark library will measure and report the timing for code within the
+`for(...)` loop.
-Alternatively, link against the `benchmark_main` library and remove
-`BENCHMARK_MAIN();` above to get the same behavior.
-
-The compiled executable will run all benchmarks by default. Pass the `--help`
-flag for option information or see the guide below.
-
-## Platform Specific Build Instructions
-
-### Building with GCC
-
+#### Platform-specific libraries
When the library is built using GCC it is necessary to link with the pthread
library due to how GCC implements `std::thread`. Failing to link to pthread will
lead to runtime exceptions (unless you're using libc++), not linker errors. See
@@ -204,88 +131,753 @@
also use `-lpthread`, but there are potential issues with ordering of command
line parameters if you use that.
-### Building with Visual Studio 2015 or 2017
-
-The `shlwapi` library (`-lshlwapi`) is required to support a call to `CPUInfo` which reads the registry. Either add `shlwapi.lib` under `[ Configuration Properties > Linker > Input ]`, or use the following:
-
-```
-// Alternatively, can add libraries using linker options.
-#ifdef _WIN32
-#pragma comment ( lib, "Shlwapi.lib" )
-#ifdef _DEBUG
-#pragma comment ( lib, "benchmarkd.lib" )
-#else
-#pragma comment ( lib, "benchmark.lib" )
-#endif
-#endif
-```
-
-Can also use the graphical version of CMake:
-* Open `CMake GUI`.
-* Under `Where to build the binaries`, same path as source plus `build`.
-* Under `CMAKE_INSTALL_PREFIX`, same path as source plus `install`.
-* Click `Configure`, `Generate`, `Open Project`.
-* If build fails, try deleting entire directory and starting again, or unticking options to build less.
-
-### Building with Intel 2015 Update 1 or Intel System Studio Update 4
-
-See instructions for building with Visual Studio. Once built, right click on the solution and change the build to Intel.
-
-### Building on Solaris
+If you're running benchmarks on Windows, the shlwapi library (`-lshlwapi`) is
+also required.
If you're running benchmarks on solaris, you'll want the kstat library linked in
too (`-lkstat`).
-## User Guide
+### Passing arguments
+Sometimes a family of benchmarks can be implemented with just one routine that
+takes an extra argument to specify which one of the family of benchmarks to
+run. For example, the following code defines a family of benchmarks for
+measuring the speed of `memcpy()` calls of different lengths:
-### Command Line
-[Output Formats](#output-formats)
+```c++
+static void BM_memcpy(benchmark::State& state) {
+ char* src = new char[state.range(0)];
+ char* dst = new char[state.range(0)];
+ memset(src, 'x', state.range(0));
+ for (auto _ : state)
+ memcpy(dst, src, state.range(0));
+ state.SetBytesProcessed(int64_t(state.iterations()) *
+ int64_t(state.range(0)));
+ delete[] src;
+ delete[] dst;
+}
+BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10);
+```
-[Output Files](#output-files)
+The preceding code is quite repetitive, and can be replaced with the following
+short-hand. The following invocation will pick a few appropriate arguments in
+the specified range and will generate a benchmark for each such argument.
-[Running a Subset of Benchmarks](#running-a-subset-of-benchmarks)
+```c++
+BENCHMARK(BM_memcpy)->Range(8, 8<<10);
+```
-[Result Comparison](#result-comparison)
+By default the arguments in the range are generated in multiples of eight and
+the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the
+range multiplier is changed to multiples of two.
-### Library
-[Runtime and Reporting Considerations](#runtime-and-reporting-considerations)
+```c++
+BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10);
+```
+Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ].
-[Passing Arguments](#passing-arguments)
+You might have a benchmark that depends on two or more inputs. For example, the
+following code defines a family of benchmarks for measuring the speed of set
+insertion.
-[Calculating Asymptotic Complexity](#asymptotic-complexity)
+```c++
+static void BM_SetInsert(benchmark::State& state) {
+ std::set<int> data;
+ for (auto _ : state) {
+ state.PauseTiming();
+ data = ConstructRandomSet(state.range(0));
+ state.ResumeTiming();
+ for (int j = 0; j < state.range(1); ++j)
+ data.insert(RandomNumber());
+ }
+}
+BENCHMARK(BM_SetInsert)
+ ->Args({1<<10, 128})
+ ->Args({2<<10, 128})
+ ->Args({4<<10, 128})
+ ->Args({8<<10, 128})
+ ->Args({1<<10, 512})
+ ->Args({2<<10, 512})
+ ->Args({4<<10, 512})
+ ->Args({8<<10, 512});
+```
-[Templated Benchmarks](#templated-benchmarks)
+The preceding code is quite repetitive, and can be replaced with the following
+short-hand. The following macro will pick a few appropriate arguments in the
+product of the two specified ranges and will generate a benchmark for each such
+pair.
-[Fixtures](#fixtures)
+```c++
+BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}});
+```
-[Custom Counters](#custom-counters)
+For more complex patterns of inputs, passing a custom function to `Apply` allows
+programmatic specification of an arbitrary set of arguments on which to run the
+benchmark. The following example enumerates a dense range on one parameter,
+and a sparse range on the second.
-[Multithreaded Benchmarks](#multithreaded-benchmarks)
+```c++
+static void CustomArguments(benchmark::internal::Benchmark* b) {
+ for (int i = 0; i <= 10; ++i)
+ for (int j = 32; j <= 1024*1024; j *= 8)
+ b->Args({i, j});
+}
+BENCHMARK(BM_SetInsert)->Apply(CustomArguments);
+```
-[CPU Timers](#cpu-timers)
+### Calculate asymptotic complexity (Big O)
+Asymptotic complexity might be calculated for a family of benchmarks. The
+following code will calculate the coefficient for the high-order term in the
+running time and the normalized root-mean square error of string comparison.
-[Manual Timing](#manual-timing)
+```c++
+static void BM_StringCompare(benchmark::State& state) {
+ std::string s1(state.range(0), '-');
+ std::string s2(state.range(0), '-');
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(s1.compare(s2));
+ }
+ state.SetComplexityN(state.range(0));
+}
+BENCHMARK(BM_StringCompare)
+ ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN);
+```
-[Setting the Time Unit](#setting-the-time-unit)
+As shown in the following invocation, asymptotic complexity might also be
+calculated automatically.
-[Preventing Optimization](#preventing-optimization)
+```c++
+BENCHMARK(BM_StringCompare)
+ ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity();
+```
-[Reporting Statistics](#reporting-statistics)
+The following code will specify asymptotic complexity with a lambda function,
+that might be used to customize high-order term calculation.
-[Custom Statistics](#custom-statistics)
+```c++
+BENCHMARK(BM_StringCompare)->RangeMultiplier(2)
+ ->Range(1<<10, 1<<18)->Complexity([](int64_t n)->double{return n; });
+```
-[Using RegisterBenchmark](#using-register-benchmark)
+### Templated benchmarks
+Templated benchmarks work the same way: This example produces and consumes
+messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the
+absence of multiprogramming.
-[Exiting with an Error](#exiting-with-an-error)
+```c++
+template <class Q> void BM_Sequential(benchmark::State& state) {
+ Q q;
+ typename Q::value_type v;
+ for (auto _ : state) {
+ for (int i = state.range(0); i--; )
+ q.push(v);
+ for (int e = state.range(0); e--; )
+ q.Wait(&v);
+ }
+ // actually messages, not bytes:
+ state.SetBytesProcessed(
+ static_cast<int64_t>(state.iterations())*state.range(0));
+}
+BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue<int>)->Range(1<<0, 1<<10);
+```
-[A Faster KeepRunning Loop](#a-faster-keep-running-loop)
+Three macros are provided for adding benchmark templates.
-[Disabling CPU Frequency Scaling](#disabling-cpu-frequency-scaling)
+```c++
+#ifdef BENCHMARK_HAS_CXX11
+#define BENCHMARK_TEMPLATE(func, ...) // Takes any number of parameters.
+#else // C++ < C++11
+#define BENCHMARK_TEMPLATE(func, arg1)
+#endif
+#define BENCHMARK_TEMPLATE1(func, arg1)
+#define BENCHMARK_TEMPLATE2(func, arg1, arg2)
+```
-<a name="output-formats" />
+### A Faster KeepRunning loop
+
+In C++11 mode, a ranged-based for loop should be used in preference to
+the `KeepRunning` loop for running the benchmarks. For example:
+
+```c++
+static void BM_Fast(benchmark::State &state) {
+ for (auto _ : state) {
+ FastOperation();
+ }
+}
+BENCHMARK(BM_Fast);
+```
+
+The reason the ranged-for loop is faster than using `KeepRunning`, is
+because `KeepRunning` requires a memory load and store of the iteration count
+ever iteration, whereas the ranged-for variant is able to keep the iteration count
+in a register.
+
+For example, an empty inner loop of using the ranged-based for method looks like:
+
+```asm
+# Loop Init
+ mov rbx, qword ptr [r14 + 104]
+ call benchmark::State::StartKeepRunning()
+ test rbx, rbx
+ je .LoopEnd
+.LoopHeader: # =>This Inner Loop Header: Depth=1
+ add rbx, -1
+ jne .LoopHeader
+.LoopEnd:
+```
+
+Compared to an empty `KeepRunning` loop, which looks like:
+
+```asm
+.LoopHeader: # in Loop: Header=BB0_3 Depth=1
+ cmp byte ptr [rbx], 1
+ jne .LoopInit
+.LoopBody: # =>This Inner Loop Header: Depth=1
+ mov rax, qword ptr [rbx + 8]
+ lea rcx, [rax + 1]
+ mov qword ptr [rbx + 8], rcx
+ cmp rax, qword ptr [rbx + 104]
+ jb .LoopHeader
+ jmp .LoopEnd
+.LoopInit:
+ mov rdi, rbx
+ call benchmark::State::StartKeepRunning()
+ jmp .LoopBody
+.LoopEnd:
+```
+
+Unless C++03 compatibility is required, the ranged-for variant of writing
+the benchmark loop should be preferred.
+
+## Passing arbitrary arguments to a benchmark
+In C++11 it is possible to define a benchmark that takes an arbitrary number
+of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)`
+macro creates a benchmark that invokes `func` with the `benchmark::State` as
+the first argument followed by the specified `args...`.
+The `test_case_name` is appended to the name of the benchmark and
+should describe the values passed.
+
+```c++
+template <class ...ExtraArgs>
+void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) {
+ [...]
+}
+// Registers a benchmark named "BM_takes_args/int_string_test" that passes
+// the specified values to `extra_args`.
+BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc"));
+```
+Note that elements of `...args` may refer to global variables. Users should
+avoid modifying global state inside of a benchmark.
+
+## Using RegisterBenchmark(name, fn, args...)
+
+The `RegisterBenchmark(name, func, args...)` function provides an alternative
+way to create and register benchmarks.
+`RegisterBenchmark(name, func, args...)` creates, registers, and returns a
+pointer to a new benchmark with the specified `name` that invokes
+`func(st, args...)` where `st` is a `benchmark::State` object.
+
+Unlike the `BENCHMARK` registration macros, which can only be used at the global
+scope, the `RegisterBenchmark` can be called anywhere. This allows for
+benchmark tests to be registered programmatically.
+
+Additionally `RegisterBenchmark` allows any callable object to be registered
+as a benchmark. Including capturing lambdas and function objects.
+
+For Example:
+```c++
+auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ };
+
+int main(int argc, char** argv) {
+ for (auto& test_input : { /* ... */ })
+ benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input);
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+}
+```
+
+### Multithreaded benchmarks
+In a multithreaded test (benchmark invoked by multiple threads simultaneously),
+it is guaranteed that none of the threads will start until all have reached
+the start of the benchmark loop, and all will have finished before any thread
+exits the benchmark loop. (This behavior is also provided by the `KeepRunning()`
+API) As such, any global setup or teardown can be wrapped in a check against the thread
+index:
+
+```c++
+static void BM_MultiThreaded(benchmark::State& state) {
+ if (state.thread_index == 0) {
+ // Setup code here.
+ }
+ for (auto _ : state) {
+ // Run the test as normal.
+ }
+ if (state.thread_index == 0) {
+ // Teardown code here.
+ }
+}
+BENCHMARK(BM_MultiThreaded)->Threads(2);
+```
+
+If the benchmarked code itself uses threads and you want to compare it to
+single-threaded code, you may want to use real-time ("wallclock") measurements
+for latency comparisons:
+
+```c++
+BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime();
+```
+
+Without `UseRealTime`, CPU time is used by default.
+
+## Controlling timers
+Normally, the entire duration of the work loop (`for (auto _ : state) {}`)
+is measured. But sometimes, it is nessesary to do some work inside of
+that loop, every iteration, but without counting that time to the benchmark time.
+That is possible, althought it is not recommended, since it has high overhead.
+
+```c++
+static void BM_SetInsert_With_Timer_Control(benchmark::State& state) {
+ std::set<int> data;
+ for (auto _ : state) {
+ state.PauseTiming(); // Stop timers. They will not count until they are resumed.
+ data = ConstructRandomSet(state.range(0)); // Do something that should not be measured
+ state.ResumeTiming(); // And resume timers. They are now counting again.
+ // The rest will be measured.
+ for (int j = 0; j < state.range(1); ++j)
+ data.insert(RandomNumber());
+ }
+}
+BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}});
+```
+
+## Manual timing
+For benchmarking something for which neither CPU time nor real-time are
+correct or accurate enough, completely manual timing is supported using
+the `UseManualTime` function.
+
+When `UseManualTime` is used, the benchmarked code must call
+`SetIterationTime` once per iteration of the benchmark loop to
+report the manually measured time.
+
+An example use case for this is benchmarking GPU execution (e.g. OpenCL
+or CUDA kernels, OpenGL or Vulkan or Direct3D draw calls), which cannot
+be accurately measured using CPU time or real-time. Instead, they can be
+measured accurately using a dedicated API, and these measurement results
+can be reported back with `SetIterationTime`.
+
+```c++
+static void BM_ManualTiming(benchmark::State& state) {
+ int microseconds = state.range(0);
+ std::chrono::duration<double, std::micro> sleep_duration {
+ static_cast<double>(microseconds)
+ };
+
+ for (auto _ : state) {
+ auto start = std::chrono::high_resolution_clock::now();
+ // Simulate some useful workload with a sleep
+ std::this_thread::sleep_for(sleep_duration);
+ auto end = std::chrono::high_resolution_clock::now();
+
+ auto elapsed_seconds =
+ std::chrono::duration_cast<std::chrono::duration<double>>(
+ end - start);
+
+ state.SetIterationTime(elapsed_seconds.count());
+ }
+}
+BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime();
+```
+
+### Preventing optimisation
+To prevent a value or expression from being optimized away by the compiler
+the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()`
+functions can be used.
+
+```c++
+static void BM_test(benchmark::State& state) {
+ for (auto _ : state) {
+ int x = 0;
+ for (int i=0; i < 64; ++i) {
+ benchmark::DoNotOptimize(x += i);
+ }
+ }
+}
+```
+
+`DoNotOptimize(<expr>)` forces the *result* of `<expr>` to be stored in either
+memory or a register. For GNU based compilers it acts as read/write barrier
+for global memory. More specifically it forces the compiler to flush pending
+writes to memory and reload any other values as necessary.
+
+Note that `DoNotOptimize(<expr>)` does not prevent optimizations on `<expr>`
+in any way. `<expr>` may even be removed entirely when the result is already
+known. For example:
+
+```c++
+ /* Example 1: `<expr>` is removed entirely. */
+ int foo(int x) { return x + 42; }
+ while (...) DoNotOptimize(foo(0)); // Optimized to DoNotOptimize(42);
+
+ /* Example 2: Result of '<expr>' is only reused */
+ int bar(int) __attribute__((const));
+ while (...) DoNotOptimize(bar(0)); // Optimized to:
+ // int __result__ = bar(0);
+ // while (...) DoNotOptimize(__result__);
+```
+
+The second tool for preventing optimizations is `ClobberMemory()`. In essence
+`ClobberMemory()` forces the compiler to perform all pending writes to global
+memory. Memory managed by block scope objects must be "escaped" using
+`DoNotOptimize(...)` before it can be clobbered. In the below example
+`ClobberMemory()` prevents the call to `v.push_back(42)` from being optimized
+away.
+
+```c++
+static void BM_vector_push_back(benchmark::State& state) {
+ for (auto _ : state) {
+ std::vector<int> v;
+ v.reserve(1);
+ benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered.
+ v.push_back(42);
+ benchmark::ClobberMemory(); // Force 42 to be written to memory.
+ }
+}
+```
+
+Note that `ClobberMemory()` is only available for GNU or MSVC based compilers.
+
+### Set time unit manually
+If a benchmark runs a few milliseconds it may be hard to visually compare the
+measured times, since the output data is given in nanoseconds per default. In
+order to manually set the time unit, you can specify it manually:
+
+```c++
+BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
+```
+
+### Reporting the mean, median and standard deviation by repeated benchmarks
+By default each benchmark is run once and that single result is reported.
+However benchmarks are often noisy and a single result may not be representative
+of the overall behavior. For this reason it's possible to repeatedly rerun the
+benchmark.
+
+The number of runs of each benchmark is specified globally by the
+`--benchmark_repetitions` flag or on a per benchmark basis by calling
+`Repetitions` on the registered benchmark object. When a benchmark is run more
+than once the mean, median and standard deviation of the runs will be reported.
+
+Additionally the `--benchmark_report_aggregates_only={true|false}`,
+`--benchmark_display_aggregates_only={true|false}` flags or
+`ReportAggregatesOnly(bool)`, `DisplayAggregatesOnly(bool)` functions can be
+used to change how repeated tests are reported. By default the result of each
+repeated run is reported. When `report aggregates only` option is `true`,
+only the aggregates (i.e. mean, median and standard deviation, maybe complexity
+measurements if they were requested) of the runs is reported, to both the
+reporters - standard output (console), and the file.
+However when only the `display aggregates only` option is `true`,
+only the aggregates are displayed in the standard output, while the file
+output still contains everything.
+Calling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a
+registered benchmark object overrides the value of the appropriate flag for that
+benchmark.
+
+## User-defined statistics for repeated benchmarks
+While having mean, median and standard deviation is nice, this may not be
+enough for everyone. For example you may want to know what is the largest
+observation, e.g. because you have some real-time constraints. This is easy.
+The following code will specify a custom statistic to be calculated, defined
+by a lambda function.
+
+```c++
+void BM_spin_empty(benchmark::State& state) {
+ for (auto _ : state) {
+ for (int x = 0; x < state.range(0); ++x) {
+ benchmark::DoNotOptimize(x);
+ }
+ }
+}
+
+BENCHMARK(BM_spin_empty)
+ ->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
+ return *(std::max_element(std::begin(v), std::end(v)));
+ })
+ ->Arg(512);
+```
+
+## Fixtures
+Fixture tests are created by
+first defining a type that derives from `::benchmark::Fixture` and then
+creating/registering the tests using the following macros:
+
+* `BENCHMARK_F(ClassName, Method)`
+* `BENCHMARK_DEFINE_F(ClassName, Method)`
+* `BENCHMARK_REGISTER_F(ClassName, Method)`
+
+For Example:
+
+```c++
+class MyFixture : public benchmark::Fixture {
+public:
+ void SetUp(const ::benchmark::State& state) {
+ }
+
+ void TearDown(const ::benchmark::State& state) {
+ }
+};
+
+BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) {
+ for (auto _ : st) {
+ ...
+ }
+}
+
+BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) {
+ for (auto _ : st) {
+ ...
+ }
+}
+/* BarTest is NOT registered */
+BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2);
+/* BarTest is now registered */
+```
+
+### Templated fixtures
+Also you can create templated fixture by using the following macros:
+
+* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)`
+* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)`
+
+For example:
+```c++
+template<typename T>
+class MyFixture : public benchmark::Fixture {};
+
+BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) {
+ for (auto _ : st) {
+ ...
+ }
+}
+
+BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) {
+ for (auto _ : st) {
+ ...
+ }
+}
+
+BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2);
+```
+
+## User-defined counters
+
+You can add your own counters with user-defined names. The example below
+will add columns "Foo", "Bar" and "Baz" in its output:
+
+```c++
+static void UserCountersExample1(benchmark::State& state) {
+ double numFoos = 0, numBars = 0, numBazs = 0;
+ for (auto _ : state) {
+ // ... count Foo,Bar,Baz events
+ }
+ state.counters["Foo"] = numFoos;
+ state.counters["Bar"] = numBars;
+ state.counters["Baz"] = numBazs;
+}
+```
+
+The `state.counters` object is a `std::map` with `std::string` keys
+and `Counter` values. The latter is a `double`-like class, via an implicit
+conversion to `double&`. Thus you can use all of the standard arithmetic
+assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter.
+
+In multithreaded benchmarks, each counter is set on the calling thread only.
+When the benchmark finishes, the counters from each thread will be summed;
+the resulting sum is the value which will be shown for the benchmark.
+
+The `Counter` constructor accepts three parameters: the value as a `double`
+; a bit flag which allows you to show counters as rates, and/or as per-thread
+iteration, and/or as per-thread averages, and/or iteration invariants;
+and a flag specifying the 'unit' - i.e. is 1k a 1000 (default,
+`benchmark::Counter::OneK::kIs1000`), or 1024
+(`benchmark::Counter::OneK::kIs1024`)?
+
+```c++
+ // sets a simple counter
+ state.counters["Foo"] = numFoos;
+
+ // Set the counter as a rate. It will be presented divided
+ // by the duration of the benchmark.
+ state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate);
+
+ // Set the counter as a thread-average quantity. It will
+ // be presented divided by the number of threads.
+ state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads);
+
+ // There's also a combined flag:
+ state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate);
+
+ // This says that we process with the rate of state.range(0) bytes every iteration:
+ state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024);
+```
+
+When you're compiling in C++11 mode or later you can use `insert()` with
+`std::initializer_list`:
+
+```c++
+ // With C++11, this can be done:
+ state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}});
+ // ... instead of:
+ state.counters["Foo"] = numFoos;
+ state.counters["Bar"] = numBars;
+ state.counters["Baz"] = numBazs;
+```
+
+### Counter reporting
+
+When using the console reporter, by default, user counters are are printed at
+the end after the table, the same way as ``bytes_processed`` and
+``items_processed``. This is best for cases in which there are few counters,
+or where there are only a couple of lines per benchmark. Here's an example of
+the default output:
+
+```
+------------------------------------------------------------------------------
+Benchmark Time CPU Iterations UserCounters...
+------------------------------------------------------------------------------
+BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8
+BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m
+BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2
+BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4
+BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8
+BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16
+BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32
+BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4
+BM_Factorial 26 ns 26 ns 26608979 40320
+BM_Factorial/real_time 26 ns 26 ns 26587936 40320
+BM_CalculatePiRange/1 16 ns 16 ns 45704255 0
+BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374
+BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746
+BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355
+```
+
+If this doesn't suit you, you can print each counter as a table column by
+passing the flag `--benchmark_counters_tabular=true` to the benchmark
+application. This is best for cases in which there are a lot of counters, or
+a lot of lines per individual benchmark. Note that this will trigger a
+reprinting of the table header any time the counter set changes between
+individual benchmarks. Here's an example of corresponding output when
+`--benchmark_counters_tabular=true` is passed:
+
+```
+---------------------------------------------------------------------------------------
+Benchmark Time CPU Iterations Bar Bat Baz Foo
+---------------------------------------------------------------------------------------
+BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8
+BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1
+BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2
+BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4
+BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8
+BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16
+BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32
+BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4
+--------------------------------------------------------------
+Benchmark Time CPU Iterations
+--------------------------------------------------------------
+BM_Factorial 26 ns 26 ns 26392245 40320
+BM_Factorial/real_time 26 ns 26 ns 26494107 40320
+BM_CalculatePiRange/1 15 ns 15 ns 45571597 0
+BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374
+BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746
+BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355
+BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184
+BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162
+BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416
+BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159
+BM_CalculatePi/threads:8 2255 ns 9943 ns 70936
+```
+Note above the additional header printed when the benchmark changes from
+``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does
+not have the same counter set as ``BM_UserCounter``.
+
+## Exiting Benchmarks in Error
+
+When errors caused by external influences, such as file I/O and network
+communication, occur within a benchmark the
+`State::SkipWithError(const char* msg)` function can be used to skip that run
+of benchmark and report the error. Note that only future iterations of the
+`KeepRunning()` are skipped. For the ranged-for version of the benchmark loop
+Users must explicitly exit the loop, otherwise all iterations will be performed.
+Users may explicitly return to exit the benchmark immediately.
+
+The `SkipWithError(...)` function may be used at any point within the benchmark,
+including before and after the benchmark loop.
+
+For example:
+
+```c++
+static void BM_test(benchmark::State& state) {
+ auto resource = GetResource();
+ if (!resource.good()) {
+ state.SkipWithError("Resource is not good!");
+ // KeepRunning() loop will not be entered.
+ }
+ for (state.KeepRunning()) {
+ auto data = resource.read_data();
+ if (!resource.good()) {
+ state.SkipWithError("Failed to read data!");
+ break; // Needed to skip the rest of the iteration.
+ }
+ do_stuff(data);
+ }
+}
+
+static void BM_test_ranged_fo(benchmark::State & state) {
+ state.SkipWithError("test will not be entered");
+ for (auto _ : state) {
+ state.SkipWithError("Failed!");
+ break; // REQUIRED to prevent all further iterations.
+ }
+}
+```
+
+## Running a subset of the benchmarks
+
+The `--benchmark_filter=<regex>` option can be used to only run the benchmarks
+which match the specified `<regex>`. For example:
+
+```bash
+$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32
+Run on (1 X 2300 MHz CPU )
+2016-06-25 19:34:24
+Benchmark Time CPU Iterations
+----------------------------------------------------
+BM_memcpy/32 11 ns 11 ns 79545455
+BM_memcpy/32k 2181 ns 2185 ns 324074
+BM_memcpy/32 12 ns 12 ns 54687500
+BM_memcpy/32k 1834 ns 1837 ns 357143
+```
+
+## Runtime and reporting considerations
+When the benchmark binary is executed, each benchmark function is run serially.
+The number of iterations to run is determined dynamically by running the
+benchmark a few times and measuring the time taken and ensuring that the
+ultimate result will be statistically stable. As such, faster benchmark
+functions will be run for more iterations than slower benchmark functions, and
+the number of iterations is thus reported.
+
+In all cases, the number of iterations for which the benchmark is run is
+governed by the amount of time the benchmark takes. Concretely, the number of
+iterations is at least one, not more than 1e9, until CPU time is greater than
+the minimum time, or the wallclock time is 5x minimum time. The minimum time is
+set per benchmark by calling `MinTime` on the registered benchmark object.
+
+Average timings are then reported over the iterations run. If multiple
+repetitions are requested using the `--benchmark_repetitions` command-line
+option, or at registration time, the benchmark function will be run several
+times and statistical results across these repetitions will also be reported.
+
+As well as the per-benchmark entries, a preamble in the report will include
+information about the machine on which the benchmarks are run.
### Output Formats
-
The library supports multiple output formats. Use the
`--benchmark_format=<console|json|csv>` flag to set the format type. `console`
is the default format.
@@ -353,875 +945,54 @@
"BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06,
```
-<a name="output-files" />
-
### Output Files
-
-Write benchmark results to a file with the `--benchmark_out=<filename>` option.
-Specify the output format with `--benchmark_out_format={json|console|csv}`. Note that Specifying
+The library supports writing the output of the benchmark to a file specified
+by `--benchmark_out=<filename>`. The format of the output can be specified
+using `--benchmark_out_format={json|console|csv}`. Specifying
`--benchmark_out` does not suppress the console output.
-<a name="running-a-subset-of-benchmarks" />
-
-### Running a Subset of Benchmarks
-
-The `--benchmark_filter=<regex>` option can be used to only run the benchmarks
-which match the specified `<regex>`. For example:
-
-```bash
-$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32
-Run on (1 X 2300 MHz CPU )
-2016-06-25 19:34:24
-Benchmark Time CPU Iterations
-----------------------------------------------------
-BM_memcpy/32 11 ns 11 ns 79545455
-BM_memcpy/32k 2181 ns 2185 ns 324074
-BM_memcpy/32 12 ns 12 ns 54687500
-BM_memcpy/32k 1834 ns 1837 ns 357143
-```
-
-<a name="result-comparison" />
-
-### Result comparison
+## Result comparison
It is possible to compare the benchmarking results. See [Additional Tooling Documentation](docs/tools.md)
-<a name="runtime-and-reporting-considerations" />
-
-### Runtime and Reporting Considerations
-
-When the benchmark binary is executed, each benchmark function is run serially.
-The number of iterations to run is determined dynamically by running the
-benchmark a few times and measuring the time taken and ensuring that the
-ultimate result will be statistically stable. As such, faster benchmark
-functions will be run for more iterations than slower benchmark functions, and
-the number of iterations is thus reported.
-
-In all cases, the number of iterations for which the benchmark is run is
-governed by the amount of time the benchmark takes. Concretely, the number of
-iterations is at least one, not more than 1e9, until CPU time is greater than
-the minimum time, or the wallclock time is 5x minimum time. The minimum time is
-set per benchmark by calling `MinTime` on the registered benchmark object.
-
-Average timings are then reported over the iterations run. If multiple
-repetitions are requested using the `--benchmark_repetitions` command-line
-option, or at registration time, the benchmark function will be run several
-times and statistical results across these repetitions will also be reported.
-
-As well as the per-benchmark entries, a preamble in the report will include
-information about the machine on which the benchmarks are run.
-
-<a name="passing-arguments" />
-
-### Passing Arguments
-
-Sometimes a family of benchmarks can be implemented with just one routine that
-takes an extra argument to specify which one of the family of benchmarks to
-run. For example, the following code defines a family of benchmarks for
-measuring the speed of `memcpy()` calls of different lengths:
-
-```c++
-static void BM_memcpy(benchmark::State& state) {
- char* src = new char[state.range(0)];
- char* dst = new char[state.range(0)];
- memset(src, 'x', state.range(0));
- for (auto _ : state)
- memcpy(dst, src, state.range(0));
- state.SetBytesProcessed(int64_t(state.iterations()) *
- int64_t(state.range(0)));
- delete[] src;
- delete[] dst;
-}
-BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10);
-```
-
-The preceding code is quite repetitive, and can be replaced with the following
-short-hand. The following invocation will pick a few appropriate arguments in
-the specified range and will generate a benchmark for each such argument.
-
-```c++
-BENCHMARK(BM_memcpy)->Range(8, 8<<10);
-```
-
-By default the arguments in the range are generated in multiples of eight and
-the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the
-range multiplier is changed to multiples of two.
-
-```c++
-BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10);
-```
-Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ].
-
-The preceding code shows a method of defining a sparse range. The following
-example shows a method of defining a dense range. It is then used to benchmark
-the performance of `std::vector` initialization for uniformly increasing sizes.
-
-```c++
-static void BM_DenseRange(benchmark::State& state) {
- for(auto _ : state) {
- std::vector<int> v(state.range(0), state.range(0));
- benchmark::DoNotOptimize(v.data());
- benchmark::ClobberMemory();
- }
-}
-BENCHMARK(BM_DenseRange)->DenseRange(0, 1024, 128);
-```
-Now arguments generated are [ 0, 128, 256, 384, 512, 640, 768, 896, 1024 ].
-
-You might have a benchmark that depends on two or more inputs. For example, the
-following code defines a family of benchmarks for measuring the speed of set
-insertion.
-
-```c++
-static void BM_SetInsert(benchmark::State& state) {
- std::set<int> data;
- for (auto _ : state) {
- state.PauseTiming();
- data = ConstructRandomSet(state.range(0));
- state.ResumeTiming();
- for (int j = 0; j < state.range(1); ++j)
- data.insert(RandomNumber());
- }
-}
-BENCHMARK(BM_SetInsert)
- ->Args({1<<10, 128})
- ->Args({2<<10, 128})
- ->Args({4<<10, 128})
- ->Args({8<<10, 128})
- ->Args({1<<10, 512})
- ->Args({2<<10, 512})
- ->Args({4<<10, 512})
- ->Args({8<<10, 512});
-```
-
-The preceding code is quite repetitive, and can be replaced with the following
-short-hand. The following macro will pick a few appropriate arguments in the
-product of the two specified ranges and will generate a benchmark for each such
-pair.
-
-```c++
-BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}});
-```
-
-For more complex patterns of inputs, passing a custom function to `Apply` allows
-programmatic specification of an arbitrary set of arguments on which to run the
-benchmark. The following example enumerates a dense range on one parameter,
-and a sparse range on the second.
-
-```c++
-static void CustomArguments(benchmark::internal::Benchmark* b) {
- for (int i = 0; i <= 10; ++i)
- for (int j = 32; j <= 1024*1024; j *= 8)
- b->Args({i, j});
-}
-BENCHMARK(BM_SetInsert)->Apply(CustomArguments);
-```
-
-#### Passing Arbitrary Arguments to a Benchmark
-
-In C++11 it is possible to define a benchmark that takes an arbitrary number
-of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)`
-macro creates a benchmark that invokes `func` with the `benchmark::State` as
-the first argument followed by the specified `args...`.
-The `test_case_name` is appended to the name of the benchmark and
-should describe the values passed.
-
-```c++
-template <class ...ExtraArgs>
-void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) {
- [...]
-}
-// Registers a benchmark named "BM_takes_args/int_string_test" that passes
-// the specified values to `extra_args`.
-BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc"));
-```
-Note that elements of `...args` may refer to global variables. Users should
-avoid modifying global state inside of a benchmark.
-
-<a name="asymptotic-complexity" />
-
-### Calculating Asymptotic Complexity (Big O)
-
-Asymptotic complexity might be calculated for a family of benchmarks. The
-following code will calculate the coefficient for the high-order term in the
-running time and the normalized root-mean square error of string comparison.
-
-```c++
-static void BM_StringCompare(benchmark::State& state) {
- std::string s1(state.range(0), '-');
- std::string s2(state.range(0), '-');
- for (auto _ : state) {
- benchmark::DoNotOptimize(s1.compare(s2));
- }
- state.SetComplexityN(state.range(0));
-}
-BENCHMARK(BM_StringCompare)
- ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN);
-```
-
-As shown in the following invocation, asymptotic complexity might also be
-calculated automatically.
-
-```c++
-BENCHMARK(BM_StringCompare)
- ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity();
-```
-
-The following code will specify asymptotic complexity with a lambda function,
-that might be used to customize high-order term calculation.
-
-```c++
-BENCHMARK(BM_StringCompare)->RangeMultiplier(2)
- ->Range(1<<10, 1<<18)->Complexity([](int64_t n)->double{return n; });
-```
-
-<a name="templated-benchmarks" />
-
-### Templated Benchmarks
-
-This example produces and consumes messages of size `sizeof(v)` `range_x`
-times. It also outputs throughput in the absence of multiprogramming.
-
-```c++
-template <class Q> void BM_Sequential(benchmark::State& state) {
- Q q;
- typename Q::value_type v;
- for (auto _ : state) {
- for (int i = state.range(0); i--; )
- q.push(v);
- for (int e = state.range(0); e--; )
- q.Wait(&v);
- }
- // actually messages, not bytes:
- state.SetBytesProcessed(
- static_cast<int64_t>(state.iterations())*state.range(0));
-}
-BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue<int>)->Range(1<<0, 1<<10);
-```
-
-Three macros are provided for adding benchmark templates.
-
-```c++
-#ifdef BENCHMARK_HAS_CXX11
-#define BENCHMARK_TEMPLATE(func, ...) // Takes any number of parameters.
-#else // C++ < C++11
-#define BENCHMARK_TEMPLATE(func, arg1)
-#endif
-#define BENCHMARK_TEMPLATE1(func, arg1)
-#define BENCHMARK_TEMPLATE2(func, arg1, arg2)
-```
-
-<a name="fixtures" />
-
-### Fixtures
-
-Fixture tests are created by first defining a type that derives from
-`::benchmark::Fixture` and then creating/registering the tests using the
-following macros:
-
-* `BENCHMARK_F(ClassName, Method)`
-* `BENCHMARK_DEFINE_F(ClassName, Method)`
-* `BENCHMARK_REGISTER_F(ClassName, Method)`
-
-For Example:
-
-```c++
-class MyFixture : public benchmark::Fixture {
-public:
- void SetUp(const ::benchmark::State& state) {
- }
-
- void TearDown(const ::benchmark::State& state) {
- }
-};
-
-BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) {
- for (auto _ : st) {
- ...
- }
-}
-
-BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) {
- for (auto _ : st) {
- ...
- }
-}
-/* BarTest is NOT registered */
-BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2);
-/* BarTest is now registered */
-```
-
-#### Templated Fixtures
-
-Also you can create templated fixture by using the following macros:
-
-* `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)`
-* `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)`
-
-For example:
-```c++
-template<typename T>
-class MyFixture : public benchmark::Fixture {};
-
-BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) {
- for (auto _ : st) {
- ...
- }
-}
-
-BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) {
- for (auto _ : st) {
- ...
- }
-}
-
-BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2);
-```
-
-<a name="custom-counters" />
-
-### Custom Counters
-
-You can add your own counters with user-defined names. The example below
-will add columns "Foo", "Bar" and "Baz" in its output:
-
-```c++
-static void UserCountersExample1(benchmark::State& state) {
- double numFoos = 0, numBars = 0, numBazs = 0;
- for (auto _ : state) {
- // ... count Foo,Bar,Baz events
- }
- state.counters["Foo"] = numFoos;
- state.counters["Bar"] = numBars;
- state.counters["Baz"] = numBazs;
-}
-```
-
-The `state.counters` object is a `std::map` with `std::string` keys
-and `Counter` values. The latter is a `double`-like class, via an implicit
-conversion to `double&`. Thus you can use all of the standard arithmetic
-assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter.
-
-In multithreaded benchmarks, each counter is set on the calling thread only.
-When the benchmark finishes, the counters from each thread will be summed;
-the resulting sum is the value which will be shown for the benchmark.
-
-The `Counter` constructor accepts three parameters: the value as a `double`
-; a bit flag which allows you to show counters as rates, and/or as per-thread
-iteration, and/or as per-thread averages, and/or iteration invariants,
-and/or finally inverting the result; and a flag specifying the 'unit' - i.e.
-is 1k a 1000 (default, `benchmark::Counter::OneK::kIs1000`), or 1024
-(`benchmark::Counter::OneK::kIs1024`)?
-
-```c++
- // sets a simple counter
- state.counters["Foo"] = numFoos;
-
- // Set the counter as a rate. It will be presented divided
- // by the duration of the benchmark.
- // Meaning: per one second, how many 'foo's are processed?
- state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate);
-
- // Set the counter as a rate. It will be presented divided
- // by the duration of the benchmark, and the result inverted.
- // Meaning: how many seconds it takes to process one 'foo'?
- state.counters["FooInvRate"] = Counter(numFoos, benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
-
- // Set the counter as a thread-average quantity. It will
- // be presented divided by the number of threads.
- state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads);
-
- // There's also a combined flag:
- state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate);
-
- // This says that we process with the rate of state.range(0) bytes every iteration:
- state.counters["BytesProcessed"] = Counter(state.range(0), benchmark::Counter::kIsIterationInvariantRate, benchmark::Counter::OneK::kIs1024);
-```
-
-When you're compiling in C++11 mode or later you can use `insert()` with
-`std::initializer_list`:
-
-```c++
- // With C++11, this can be done:
- state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}});
- // ... instead of:
- state.counters["Foo"] = numFoos;
- state.counters["Bar"] = numBars;
- state.counters["Baz"] = numBazs;
-```
-
-#### Counter Reporting
-
-When using the console reporter, by default, user counters are printed at
-the end after the table, the same way as ``bytes_processed`` and
-``items_processed``. This is best for cases in which there are few counters,
-or where there are only a couple of lines per benchmark. Here's an example of
-the default output:
+## Debug vs Release
+By default, benchmark builds as a debug library. You will see a warning in the
+output when this is the case. To build it as a release library instead, use:
```
-------------------------------------------------------------------------------
-Benchmark Time CPU Iterations UserCounters...
-------------------------------------------------------------------------------
-BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8
-BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m
-BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2
-BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4
-BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8
-BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16
-BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32
-BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4
-BM_Factorial 26 ns 26 ns 26608979 40320
-BM_Factorial/real_time 26 ns 26 ns 26587936 40320
-BM_CalculatePiRange/1 16 ns 16 ns 45704255 0
-BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374
-BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746
-BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355
+cmake -DCMAKE_BUILD_TYPE=Release
```
-If this doesn't suit you, you can print each counter as a table column by
-passing the flag `--benchmark_counters_tabular=true` to the benchmark
-application. This is best for cases in which there are a lot of counters, or
-a lot of lines per individual benchmark. Note that this will trigger a
-reprinting of the table header any time the counter set changes between
-individual benchmarks. Here's an example of corresponding output when
-`--benchmark_counters_tabular=true` is passed:
+To enable link-time optimisation, use
```
----------------------------------------------------------------------------------------
-Benchmark Time CPU Iterations Bar Bat Baz Foo
----------------------------------------------------------------------------------------
-BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8
-BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1
-BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2
-BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4
-BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8
-BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16
-BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32
-BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4
---------------------------------------------------------------
-Benchmark Time CPU Iterations
---------------------------------------------------------------
-BM_Factorial 26 ns 26 ns 26392245 40320
-BM_Factorial/real_time 26 ns 26 ns 26494107 40320
-BM_CalculatePiRange/1 15 ns 15 ns 45571597 0
-BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374
-BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746
-BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355
-BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184
-BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162
-BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416
-BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159
-BM_CalculatePi/threads:8 2255 ns 9943 ns 70936
-```
-Note above the additional header printed when the benchmark changes from
-``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does
-not have the same counter set as ``BM_UserCounter``.
-
-<a name="multithreaded-benchmarks"/>
-
-### Multithreaded Benchmarks
-
-In a multithreaded test (benchmark invoked by multiple threads simultaneously),
-it is guaranteed that none of the threads will start until all have reached
-the start of the benchmark loop, and all will have finished before any thread
-exits the benchmark loop. (This behavior is also provided by the `KeepRunning()`
-API) As such, any global setup or teardown can be wrapped in a check against the thread
-index:
-
-```c++
-static void BM_MultiThreaded(benchmark::State& state) {
- if (state.thread_index == 0) {
- // Setup code here.
- }
- for (auto _ : state) {
- // Run the test as normal.
- }
- if (state.thread_index == 0) {
- // Teardown code here.
- }
-}
-BENCHMARK(BM_MultiThreaded)->Threads(2);
+cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true
```
-If the benchmarked code itself uses threads and you want to compare it to
-single-threaded code, you may want to use real-time ("wallclock") measurements
-for latency comparisons:
+If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake
+cache variables, if autodetection fails.
-```c++
-BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime();
-```
+If you are using clang, you may need to set `LLVMAR_EXECUTABLE`,
+`LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables.
-Without `UseRealTime`, CPU time is used by default.
+## Compiler Support
-<a name="cpu-timers" />
+Google Benchmark uses C++11 when building the library. As such we require
+a modern C++ toolchain, both compiler and standard library.
-### CPU Timers
+The following minimum versions are strongly recommended build the library:
-By default, the CPU timer only measures the time spent by the main thread.
-If the benchmark itself uses threads internally, this measurement may not
-be what you are looking for. Instead, there is a way to measure the total
-CPU usage of the process, by all the threads.
+* GCC 4.8
+* Clang 3.4
+* Visual Studio 2013
+* Intel 2015 Update 1
-```c++
-void callee(int i);
+Anything older *may* work.
-static void MyMain(int size) {
-#pragma omp parallel for
- for(int i = 0; i < size; i++)
- callee(i);
-}
+Note: Using the library and its headers in C++03 is supported. C++11 is only
+required to build the library.
-static void BM_OpenMP(benchmark::State& state) {
- for (auto _ : state)
- MyMain(state.range(0));
-}
-
-// Measure the time spent by the main thread, use it to decide for how long to
-// run the benchmark loop. Depending on the internal implementation detail may
-// measure to anywhere from near-zero (the overhead spent before/after work
-// handoff to worker thread[s]) to the whole single-thread time.
-BENCHMARK(BM_OpenMP)->Range(8, 8<<10);
-
-// Measure the user-visible time, the wall clock (literally, the time that
-// has passed on the clock on the wall), use it to decide for how long to
-// run the benchmark loop. This will always be meaningful, an will match the
-// time spent by the main thread in single-threaded case, in general decreasing
-// with the number of internal threads doing the work.
-BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->UseRealTime();
-
-// Measure the total CPU consumption, use it to decide for how long to
-// run the benchmark loop. This will always measure to no less than the
-// time spent by the main thread in single-threaded case.
-BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime();
-
-// A mixture of the last two. Measure the total CPU consumption, but use the
-// wall clock to decide for how long to run the benchmark loop.
-BENCHMARK(BM_OpenMP)->Range(8, 8<<10)->MeasureProcessCPUTime()->UseRealTime();
-```
-
-#### Controlling Timers
-
-Normally, the entire duration of the work loop (`for (auto _ : state) {}`)
-is measured. But sometimes, it is necessary to do some work inside of
-that loop, every iteration, but without counting that time to the benchmark time.
-That is possible, although it is not recommended, since it has high overhead.
-
-```c++
-static void BM_SetInsert_With_Timer_Control(benchmark::State& state) {
- std::set<int> data;
- for (auto _ : state) {
- state.PauseTiming(); // Stop timers. They will not count until they are resumed.
- data = ConstructRandomSet(state.range(0)); // Do something that should not be measured
- state.ResumeTiming(); // And resume timers. They are now counting again.
- // The rest will be measured.
- for (int j = 0; j < state.range(1); ++j)
- data.insert(RandomNumber());
- }
-}
-BENCHMARK(BM_SetInsert_With_Timer_Control)->Ranges({{1<<10, 8<<10}, {128, 512}});
-```
-
-<a name="manual-timing" />
-
-### Manual Timing
-
-For benchmarking something for which neither CPU time nor real-time are
-correct or accurate enough, completely manual timing is supported using
-the `UseManualTime` function.
-
-When `UseManualTime` is used, the benchmarked code must call
-`SetIterationTime` once per iteration of the benchmark loop to
-report the manually measured time.
-
-An example use case for this is benchmarking GPU execution (e.g. OpenCL
-or CUDA kernels, OpenGL or Vulkan or Direct3D draw calls), which cannot
-be accurately measured using CPU time or real-time. Instead, they can be
-measured accurately using a dedicated API, and these measurement results
-can be reported back with `SetIterationTime`.
-
-```c++
-static void BM_ManualTiming(benchmark::State& state) {
- int microseconds = state.range(0);
- std::chrono::duration<double, std::micro> sleep_duration {
- static_cast<double>(microseconds)
- };
-
- for (auto _ : state) {
- auto start = std::chrono::high_resolution_clock::now();
- // Simulate some useful workload with a sleep
- std::this_thread::sleep_for(sleep_duration);
- auto end = std::chrono::high_resolution_clock::now();
-
- auto elapsed_seconds =
- std::chrono::duration_cast<std::chrono::duration<double>>(
- end - start);
-
- state.SetIterationTime(elapsed_seconds.count());
- }
-}
-BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime();
-```
-
-<a name="setting-the-time-unit" />
-
-### Setting the Time Unit
-
-If a benchmark runs a few milliseconds it may be hard to visually compare the
-measured times, since the output data is given in nanoseconds per default. In
-order to manually set the time unit, you can specify it manually:
-
-```c++
-BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
-```
-
-<a name="preventing-optimization" />
-
-### Preventing Optimization
-
-To prevent a value or expression from being optimized away by the compiler
-the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()`
-functions can be used.
-
-```c++
-static void BM_test(benchmark::State& state) {
- for (auto _ : state) {
- int x = 0;
- for (int i=0; i < 64; ++i) {
- benchmark::DoNotOptimize(x += i);
- }
- }
-}
-```
-
-`DoNotOptimize(<expr>)` forces the *result* of `<expr>` to be stored in either
-memory or a register. For GNU based compilers it acts as read/write barrier
-for global memory. More specifically it forces the compiler to flush pending
-writes to memory and reload any other values as necessary.
-
-Note that `DoNotOptimize(<expr>)` does not prevent optimizations on `<expr>`
-in any way. `<expr>` may even be removed entirely when the result is already
-known. For example:
-
-```c++
- /* Example 1: `<expr>` is removed entirely. */
- int foo(int x) { return x + 42; }
- while (...) DoNotOptimize(foo(0)); // Optimized to DoNotOptimize(42);
-
- /* Example 2: Result of '<expr>' is only reused */
- int bar(int) __attribute__((const));
- while (...) DoNotOptimize(bar(0)); // Optimized to:
- // int __result__ = bar(0);
- // while (...) DoNotOptimize(__result__);
-```
-
-The second tool for preventing optimizations is `ClobberMemory()`. In essence
-`ClobberMemory()` forces the compiler to perform all pending writes to global
-memory. Memory managed by block scope objects must be "escaped" using
-`DoNotOptimize(...)` before it can be clobbered. In the below example
-`ClobberMemory()` prevents the call to `v.push_back(42)` from being optimized
-away.
-
-```c++
-static void BM_vector_push_back(benchmark::State& state) {
- for (auto _ : state) {
- std::vector<int> v;
- v.reserve(1);
- benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered.
- v.push_back(42);
- benchmark::ClobberMemory(); // Force 42 to be written to memory.
- }
-}
-```
-
-Note that `ClobberMemory()` is only available for GNU or MSVC based compilers.
-
-<a name="reporting-statistics" />
-
-### Statistics: Reporting the Mean, Median and Standard Deviation of Repeated Benchmarks
-
-By default each benchmark is run once and that single result is reported.
-However benchmarks are often noisy and a single result may not be representative
-of the overall behavior. For this reason it's possible to repeatedly rerun the
-benchmark.
-
-The number of runs of each benchmark is specified globally by the
-`--benchmark_repetitions` flag or on a per benchmark basis by calling
-`Repetitions` on the registered benchmark object. When a benchmark is run more
-than once the mean, median and standard deviation of the runs will be reported.
-
-Additionally the `--benchmark_report_aggregates_only={true|false}`,
-`--benchmark_display_aggregates_only={true|false}` flags or
-`ReportAggregatesOnly(bool)`, `DisplayAggregatesOnly(bool)` functions can be
-used to change how repeated tests are reported. By default the result of each
-repeated run is reported. When `report aggregates only` option is `true`,
-only the aggregates (i.e. mean, median and standard deviation, maybe complexity
-measurements if they were requested) of the runs is reported, to both the
-reporters - standard output (console), and the file.
-However when only the `display aggregates only` option is `true`,
-only the aggregates are displayed in the standard output, while the file
-output still contains everything.
-Calling `ReportAggregatesOnly(bool)` / `DisplayAggregatesOnly(bool)` on a
-registered benchmark object overrides the value of the appropriate flag for that
-benchmark.
-
-<a name="custom-statistics" />
-
-### Custom Statistics
-
-While having mean, median and standard deviation is nice, this may not be
-enough for everyone. For example you may want to know what the largest
-observation is, e.g. because you have some real-time constraints. This is easy.
-The following code will specify a custom statistic to be calculated, defined
-by a lambda function.
-
-```c++
-void BM_spin_empty(benchmark::State& state) {
- for (auto _ : state) {
- for (int x = 0; x < state.range(0); ++x) {
- benchmark::DoNotOptimize(x);
- }
- }
-}
-
-BENCHMARK(BM_spin_empty)
- ->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
- return *(std::max_element(std::begin(v), std::end(v)));
- })
- ->Arg(512);
-```
-
-<a name="using-register-benchmark" />
-
-### Using RegisterBenchmark(name, fn, args...)
-
-The `RegisterBenchmark(name, func, args...)` function provides an alternative
-way to create and register benchmarks.
-`RegisterBenchmark(name, func, args...)` creates, registers, and returns a
-pointer to a new benchmark with the specified `name` that invokes
-`func(st, args...)` where `st` is a `benchmark::State` object.
-
-Unlike the `BENCHMARK` registration macros, which can only be used at the global
-scope, the `RegisterBenchmark` can be called anywhere. This allows for
-benchmark tests to be registered programmatically.
-
-Additionally `RegisterBenchmark` allows any callable object to be registered
-as a benchmark. Including capturing lambdas and function objects.
-
-For Example:
-```c++
-auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ };
-
-int main(int argc, char** argv) {
- for (auto& test_input : { /* ... */ })
- benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input);
- benchmark::Initialize(&argc, argv);
- benchmark::RunSpecifiedBenchmarks();
-}
-```
-
-<a name="exiting-with-an-error" />
-
-### Exiting with an Error
-
-When errors caused by external influences, such as file I/O and network
-communication, occur within a benchmark the
-`State::SkipWithError(const char* msg)` function can be used to skip that run
-of benchmark and report the error. Note that only future iterations of the
-`KeepRunning()` are skipped. For the ranged-for version of the benchmark loop
-Users must explicitly exit the loop, otherwise all iterations will be performed.
-Users may explicitly return to exit the benchmark immediately.
-
-The `SkipWithError(...)` function may be used at any point within the benchmark,
-including before and after the benchmark loop.
-
-For example:
-
-```c++
-static void BM_test(benchmark::State& state) {
- auto resource = GetResource();
- if (!resource.good()) {
- state.SkipWithError("Resource is not good!");
- // KeepRunning() loop will not be entered.
- }
- while (state.KeepRunning()) {
- auto data = resource.read_data();
- if (!resource.good()) {
- state.SkipWithError("Failed to read data!");
- break; // Needed to skip the rest of the iteration.
- }
- do_stuff(data);
- }
-}
-
-static void BM_test_ranged_fo(benchmark::State & state) {
- state.SkipWithError("test will not be entered");
- for (auto _ : state) {
- state.SkipWithError("Failed!");
- break; // REQUIRED to prevent all further iterations.
- }
-}
-```
-<a name="a-faster-keep-running-loop" />
-
-### A Faster KeepRunning Loop
-
-In C++11 mode, a ranged-based for loop should be used in preference to
-the `KeepRunning` loop for running the benchmarks. For example:
-
-```c++
-static void BM_Fast(benchmark::State &state) {
- for (auto _ : state) {
- FastOperation();
- }
-}
-BENCHMARK(BM_Fast);
-```
-
-The reason the ranged-for loop is faster than using `KeepRunning`, is
-because `KeepRunning` requires a memory load and store of the iteration count
-ever iteration, whereas the ranged-for variant is able to keep the iteration count
-in a register.
-
-For example, an empty inner loop of using the ranged-based for method looks like:
-
-```asm
-# Loop Init
- mov rbx, qword ptr [r14 + 104]
- call benchmark::State::StartKeepRunning()
- test rbx, rbx
- je .LoopEnd
-.LoopHeader: # =>This Inner Loop Header: Depth=1
- add rbx, -1
- jne .LoopHeader
-.LoopEnd:
-```
-
-Compared to an empty `KeepRunning` loop, which looks like:
-
-```asm
-.LoopHeader: # in Loop: Header=BB0_3 Depth=1
- cmp byte ptr [rbx], 1
- jne .LoopInit
-.LoopBody: # =>This Inner Loop Header: Depth=1
- mov rax, qword ptr [rbx + 8]
- lea rcx, [rax + 1]
- mov qword ptr [rbx + 8], rcx
- cmp rax, qword ptr [rbx + 104]
- jb .LoopHeader
- jmp .LoopEnd
-.LoopInit:
- mov rdi, rbx
- call benchmark::State::StartKeepRunning()
- jmp .LoopBody
-.LoopEnd:
-```
-
-Unless C++03 compatibility is required, the ranged-for variant of writing
-the benchmark loop should be preferred.
-
-<a name="disabling-cpu-frequency-scaling" />
-
-### Disabling CPU Frequency Scaling
+## Disable CPU frequency scaling
If you see this error:
```
***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead.
diff --git a/WORKSPACE b/WORKSPACE
index 8df248a..54734f1 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,15 +1,7 @@
workspace(name = "com_github_google_benchmark")
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
http_archive(
- name = "rules_cc",
- strip_prefix = "rules_cc-a508235df92e71d537fcbae0c7c952ea6957a912",
- urls = ["https://github.com/bazelbuild/rules_cc/archive/a508235df92e71d537fcbae0c7c952ea6957a912.zip"],
-)
-
-http_archive(
- name = "com_google_googletest",
- strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e",
- urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"],
+ name = "com_google_googletest",
+ urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"],
+ strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e",
)
diff --git a/_config.yml b/_config.yml
deleted file mode 100644
index 1885487..0000000
--- a/_config.yml
+++ /dev/null
@@ -1 +0,0 @@
-theme: jekyll-theme-midnight
\ No newline at end of file
diff --git a/appveyor.yml b/appveyor.yml
index 81da955..cf24019 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -41,7 +41,7 @@
- cmake --build . --config %configuration%
test_script:
- - ctest --build-config %configuration% --timeout 300 --output-on-failure
+ - ctest -c %configuration% --timeout 300 --output-on-failure
artifacts:
- path: '_build/CMakeFiles/*.log'
diff --git a/cmake/CXXFeatureCheck.cmake b/cmake/CXXFeatureCheck.cmake
index 059d510..99b56dd 100644
--- a/cmake/CXXFeatureCheck.cmake
+++ b/cmake/CXXFeatureCheck.cmake
@@ -37,9 +37,9 @@
if(COMPILE_${FEATURE})
message(WARNING
"If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0")
- set(RUN_${FEATURE} 0 CACHE INTERNAL "")
+ set(RUN_${FEATURE} 0)
else()
- set(RUN_${FEATURE} 1 CACHE INTERNAL "")
+ set(RUN_${FEATURE} 1)
endif()
else()
message(STATUS "Performing Test ${FEATURE}")
diff --git a/cmake/GoogleTest.cmake b/cmake/GoogleTest.cmake
deleted file mode 100644
index dd611fc..0000000
--- a/cmake/GoogleTest.cmake
+++ /dev/null
@@ -1,41 +0,0 @@
-# Download and unpack googletest at configure time
-set(GOOGLETEST_PREFIX "${benchmark_BINARY_DIR}/third_party/googletest")
-configure_file(${benchmark_SOURCE_DIR}/cmake/GoogleTest.cmake.in ${GOOGLETEST_PREFIX}/CMakeLists.txt @ONLY)
-
-set(GOOGLETEST_PATH "${CMAKE_CURRENT_SOURCE_DIR}/googletest" CACHE PATH "") # Mind the quotes
-execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}"
- -DALLOW_DOWNLOADING_GOOGLETEST=${BENCHMARK_DOWNLOAD_DEPENDENCIES} -DGOOGLETEST_PATH:PATH=${GOOGLETEST_PATH} .
- RESULT_VARIABLE result
- WORKING_DIRECTORY ${GOOGLETEST_PREFIX}
-)
-
-if(result)
- message(FATAL_ERROR "CMake step for googletest failed: ${result}")
-endif()
-
-execute_process(
- COMMAND ${CMAKE_COMMAND} --build .
- RESULT_VARIABLE result
- WORKING_DIRECTORY ${GOOGLETEST_PREFIX}
-)
-
-if(result)
- message(FATAL_ERROR "Build step for googletest failed: ${result}")
-endif()
-
-# Prevent overriding the parent project's compiler/linker
-# settings on Windows
-set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
-
-include(${GOOGLETEST_PREFIX}/googletest-paths.cmake)
-
-# Add googletest directly to our build. This defines
-# the gtest and gtest_main targets.
-add_subdirectory(${GOOGLETEST_SOURCE_DIR}
- ${GOOGLETEST_BINARY_DIR}
- EXCLUDE_FROM_ALL)
-
-set_target_properties(gtest PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gtest,INTERFACE_INCLUDE_DIRECTORIES>)
-set_target_properties(gtest_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gtest_main,INTERFACE_INCLUDE_DIRECTORIES>)
-set_target_properties(gmock PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gmock,INTERFACE_INCLUDE_DIRECTORIES>)
-set_target_properties(gmock_main PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES $<TARGET_PROPERTY:gmock_main,INTERFACE_INCLUDE_DIRECTORIES>)
diff --git a/cmake/GoogleTest.cmake.in b/cmake/GoogleTest.cmake.in
deleted file mode 100644
index 28818ee..0000000
--- a/cmake/GoogleTest.cmake.in
+++ /dev/null
@@ -1,58 +0,0 @@
-cmake_minimum_required(VERSION 2.8.12)
-
-project(googletest-download NONE)
-
-# Enable ExternalProject CMake module
-include(ExternalProject)
-
-option(ALLOW_DOWNLOADING_GOOGLETEST "If googletest src tree is not found in location specified by GOOGLETEST_PATH, do fetch the archive from internet" OFF)
-set(GOOGLETEST_PATH "/usr/src/googletest" CACHE PATH
- "Path to the googletest root tree. Should contain googletest and googlemock subdirs. And CMakeLists.txt in root, and in both of these subdirs")
-
-# Download and install GoogleTest
-
-message(STATUS "Looking for Google Test sources")
-message(STATUS "Looking for Google Test sources in ${GOOGLETEST_PATH}")
-if(EXISTS "${GOOGLETEST_PATH}" AND IS_DIRECTORY "${GOOGLETEST_PATH}" AND EXISTS "${GOOGLETEST_PATH}/CMakeLists.txt" AND
- EXISTS "${GOOGLETEST_PATH}/googletest" AND IS_DIRECTORY "${GOOGLETEST_PATH}/googletest" AND EXISTS "${GOOGLETEST_PATH}/googletest/CMakeLists.txt" AND
- EXISTS "${GOOGLETEST_PATH}/googlemock" AND IS_DIRECTORY "${GOOGLETEST_PATH}/googlemock" AND EXISTS "${GOOGLETEST_PATH}/googlemock/CMakeLists.txt")
- message(STATUS "Found Google Test in ${GOOGLETEST_PATH}")
-
- ExternalProject_Add(
- googletest
- PREFIX "${CMAKE_BINARY_DIR}"
- DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download"
- SOURCE_DIR "${GOOGLETEST_PATH}" # use existing src dir.
- BINARY_DIR "${CMAKE_BINARY_DIR}/build"
- CONFIGURE_COMMAND ""
- BUILD_COMMAND ""
- INSTALL_COMMAND ""
- TEST_COMMAND ""
- )
-else()
- if(NOT ALLOW_DOWNLOADING_GOOGLETEST)
- message(SEND_ERROR "Did not find Google Test sources! Either pass correct path in GOOGLETEST_PATH, or enable ALLOW_DOWNLOADING_GOOGLETEST, or disable BENCHMARK_ENABLE_GTEST_TESTS / BENCHMARK_ENABLE_TESTING.")
- else()
- message(WARNING "Did not find Google Test sources! Fetching from web...")
- ExternalProject_Add(
- googletest
- GIT_REPOSITORY https://github.com/google/googletest.git
- GIT_TAG master
- PREFIX "${CMAKE_BINARY_DIR}"
- STAMP_DIR "${CMAKE_BINARY_DIR}/stamp"
- DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/download"
- SOURCE_DIR "${CMAKE_BINARY_DIR}/src"
- BINARY_DIR "${CMAKE_BINARY_DIR}/build"
- CONFIGURE_COMMAND ""
- BUILD_COMMAND ""
- INSTALL_COMMAND ""
- TEST_COMMAND ""
- )
- endif()
-endif()
-
-ExternalProject_Get_Property(googletest SOURCE_DIR BINARY_DIR)
-file(WRITE googletest-paths.cmake
-"set(GOOGLETEST_SOURCE_DIR \"${SOURCE_DIR}\")
-set(GOOGLETEST_BINARY_DIR \"${BINARY_DIR}\")
-")
diff --git a/cmake/HandleGTest.cmake b/cmake/HandleGTest.cmake
new file mode 100644
index 0000000..b9c1443
--- /dev/null
+++ b/cmake/HandleGTest.cmake
@@ -0,0 +1,113 @@
+
+include(split_list)
+
+macro(build_external_gtest)
+ include(ExternalProject)
+ set(GTEST_FLAGS "")
+ if (BENCHMARK_USE_LIBCXX)
+ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+ list(APPEND GTEST_FLAGS -stdlib=libc++)
+ else()
+ message(WARNING "Unsupported compiler (${CMAKE_CXX_COMPILER}) when using libc++")
+ endif()
+ endif()
+ if (BENCHMARK_BUILD_32_BITS)
+ list(APPEND GTEST_FLAGS -m32)
+ endif()
+ if (NOT "${CMAKE_CXX_FLAGS}" STREQUAL "")
+ list(APPEND GTEST_FLAGS ${CMAKE_CXX_FLAGS})
+ endif()
+ string(TOUPPER "${CMAKE_BUILD_TYPE}" GTEST_BUILD_TYPE)
+ if ("${GTEST_BUILD_TYPE}" STREQUAL "COVERAGE")
+ set(GTEST_BUILD_TYPE "DEBUG")
+ endif()
+ # FIXME: Since 10/Feb/2017 the googletest trunk has had a bug where
+ # -Werror=unused-function fires during the build on OS X. This is a temporary
+ # workaround to keep our travis bots from failing. It should be removed
+ # once gtest is fixed.
+ if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
+ list(APPEND GTEST_FLAGS "-Wno-unused-function")
+ endif()
+ split_list(GTEST_FLAGS)
+ set(EXCLUDE_FROM_ALL_OPT "")
+ set(EXCLUDE_FROM_ALL_VALUE "")
+ if (${CMAKE_VERSION} VERSION_GREATER "3.0.99")
+ set(EXCLUDE_FROM_ALL_OPT "EXCLUDE_FROM_ALL")
+ set(EXCLUDE_FROM_ALL_VALUE "ON")
+ endif()
+ ExternalProject_Add(googletest
+ ${EXCLUDE_FROM_ALL_OPT} ${EXCLUDE_FROM_ALL_VALUE}
+ GIT_REPOSITORY https://github.com/google/googletest.git
+ GIT_TAG master
+ PREFIX "${CMAKE_BINARY_DIR}/googletest"
+ INSTALL_DIR "${CMAKE_BINARY_DIR}/googletest"
+ CMAKE_CACHE_ARGS
+ -DCMAKE_BUILD_TYPE:STRING=${GTEST_BUILD_TYPE}
+ -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER}
+ -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}
+ -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+ -DCMAKE_INSTALL_LIBDIR:PATH=<INSTALL_DIR>/lib
+ -DCMAKE_CXX_FLAGS:STRING=${GTEST_FLAGS}
+ -Dgtest_force_shared_crt:BOOL=ON
+ )
+
+ ExternalProject_Get_Property(googletest install_dir)
+ set(GTEST_INCLUDE_DIRS ${install_dir}/include)
+ file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIRS})
+
+ set(LIB_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(LIB_PREFIX "${CMAKE_STATIC_LIBRARY_PREFIX}")
+ if("${GTEST_BUILD_TYPE}" STREQUAL "DEBUG")
+ set(LIB_SUFFIX "d${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ endif()
+
+ # Use gmock_main instead of gtest_main because it initializes gtest as well.
+ # Note: The libraries are listed in reverse order of their dependancies.
+ foreach(LIB gtest gmock gmock_main)
+ add_library(${LIB} UNKNOWN IMPORTED)
+ set_target_properties(${LIB} PROPERTIES
+ IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}${LIB}${LIB_SUFFIX}
+ INTERFACE_INCLUDE_DIRECTORIES ${GTEST_INCLUDE_DIRS}
+ INTERFACE_LINK_LIBRARIES "${GTEST_BOTH_LIBRARIES}"
+ )
+ add_dependencies(${LIB} googletest)
+ list(APPEND GTEST_BOTH_LIBRARIES ${LIB})
+ endforeach()
+endmacro(build_external_gtest)
+
+if (BENCHMARK_ENABLE_GTEST_TESTS)
+ if (IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/googletest)
+ set(GTEST_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/googletest")
+ set(INSTALL_GTEST OFF CACHE INTERNAL "")
+ set(INSTALL_GMOCK OFF CACHE INTERNAL "")
+ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/googletest)
+ set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main)
+ foreach(HEADER test mock)
+ # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we
+ # have to add the paths ourselves.
+ set(HFILE g${HEADER}/g${HEADER}.h)
+ set(HPATH ${GTEST_ROOT}/google${HEADER}/include)
+ find_path(HEADER_PATH_${HEADER} ${HFILE}
+ NO_DEFAULT_PATHS
+ HINTS ${HPATH}
+ )
+ if (NOT HEADER_PATH_${HEADER})
+ message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}")
+ endif()
+ list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}})
+ endforeach()
+ elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES)
+ build_external_gtest()
+ else()
+ find_package(GTest REQUIRED)
+ find_path(GMOCK_INCLUDE_DIRS gmock/gmock.h
+ HINTS ${GTEST_INCLUDE_DIRS})
+ if (NOT GMOCK_INCLUDE_DIRS)
+ message(FATAL_ERROR "Failed to find header gmock/gmock.h with hint ${GTEST_INCLUDE_DIRS}")
+ endif()
+ set(GTEST_INCLUDE_DIRS ${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS})
+ # FIXME: We don't currently require the gmock library to build the tests,
+ # and it's likely we won't find it, so we don't try. As long as we've
+ # found the gmock/gmock.h header and gtest_main that should be good enough.
+ endif()
+endif()
diff --git a/dependencies.md b/dependencies.md
deleted file mode 100644
index 6289b4e..0000000
--- a/dependencies.md
+++ /dev/null
@@ -1,18 +0,0 @@
-# Build tool dependency policy
-
-To ensure the broadest compatibility when building the benchmark library, but
-still allow forward progress, we require any build tooling to be available for:
-
-* Debian stable AND
-* The last two Ubuntu LTS releases AND
-
-Currently, this means using build tool versions that are available for Ubuntu
-16.04 (Xenial), Ubuntu 18.04 (Bionic), and Debian stretch.
-
-_Note, [travis](.travis.yml) runs under Ubuntu 14.04 (Trusty) for linux builds._
-
-## cmake
-The current supported version is cmake 3.5.1 as of 2018-06-06.
-
-_Note, this version is also available for Ubuntu 14.04, the previous Ubuntu LTS
-release, as `cmake3`._
diff --git a/docs/_config.yml b/docs/_config.yml
deleted file mode 100644
index 1885487..0000000
--- a/docs/_config.yml
+++ /dev/null
@@ -1 +0,0 @@
-theme: jekyll-theme-midnight
\ No newline at end of file
diff --git a/include/benchmark/benchmark.h b/include/benchmark/benchmark.h
index 144e212..e4f3921 100644
--- a/include/benchmark/benchmark.h
+++ b/include/benchmark/benchmark.h
@@ -56,7 +56,8 @@
memset(src, 'x', state.range(0));
for (auto _ : state)
memcpy(dst, src, state.range(0));
- state.SetBytesProcessed(state.iterations() * state.range(0));
+ state.SetBytesProcessed(int64_t(state.iterations()) *
+ int64_t(state.range(0)));
delete[] src; delete[] dst;
}
BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10);
@@ -121,7 +122,8 @@
q.Wait(&v);
}
// actually messages, not bytes:
- state.SetBytesProcessed(state.iterations() * state.range(0));
+ state.SetBytesProcessed(
+ static_cast<int64_t>(state.iterations())*state.range(0));
}
BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue<int>)->Range(1<<0, 1<<10);
@@ -244,11 +246,11 @@
#endif
#if defined(__GNUC__) || __has_builtin(__builtin_unreachable)
-#define BENCHMARK_UNREACHABLE() __builtin_unreachable()
+ #define BENCHMARK_UNREACHABLE() __builtin_unreachable()
#elif defined(_MSC_VER)
-#define BENCHMARK_UNREACHABLE() __assume(false)
+ #define BENCHMARK_UNREACHABLE() __assume(false)
#else
-#define BENCHMARK_UNREACHABLE() ((void)0)
+ #define BENCHMARK_UNREACHABLE() ((void)0)
#endif
namespace benchmark {
@@ -368,10 +370,7 @@
// It will be presented divided by the number of iterations.
kAvgIterations = 1U << 3U,
// Mark the counter as a iteration-average rate. See above.
- kAvgIterationsRate = kIsRate | kAvgIterations,
-
- // In the end, invert the result. This is always done last!
- kInvert = 1U << 31U
+ kAvgIterationsRate = kIsRate | kAvgIterations
};
enum OneK {
@@ -414,11 +413,9 @@
// calculated automatically to the best fit.
enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda };
-typedef uint64_t IterationCount;
-
// BigOFunc is passed to a benchmark in order to specify the asymptotic
// computational complexity for the benchmark.
-typedef double(BigOFunc)(IterationCount);
+typedef double(BigOFunc)(int64_t);
// StatisticsFunc is passed to a benchmark in order to compute some descriptive
// statistics over all the measurements of some type
@@ -491,7 +488,7 @@
// while (state.KeepRunningBatch(1000)) {
// // process 1000 elements
// }
- bool KeepRunningBatch(IterationCount n);
+ bool KeepRunningBatch(size_t n);
// REQUIRES: timer is running and 'SkipWithError(...)' has not been called
// by the current thread.
@@ -577,7 +574,7 @@
void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; }
BENCHMARK_ALWAYS_INLINE
- int64_t complexity_length_n() const { return complexity_n_; }
+ int64_t complexity_length_n() { return complexity_n_; }
// If this routine is called with items > 0, then an items/s
// label is printed on the benchmark report line for the currently
@@ -630,7 +627,7 @@
int64_t range_y() const { return range(1); }
BENCHMARK_ALWAYS_INLINE
- IterationCount iterations() const {
+ size_t iterations() const {
if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) {
return 0;
}
@@ -641,15 +638,15 @@
: // items we expect on the first cache line (ie 64 bytes of the struct)
// When total_iterations_ is 0, KeepRunning() and friends will return false.
// May be larger than max_iterations.
- IterationCount total_iterations_;
+ size_t total_iterations_;
// When using KeepRunningBatch(), batch_leftover_ holds the number of
// iterations beyond max_iters that were run. Used to track
// completed_iterations_ accurately.
- IterationCount batch_leftover_;
+ size_t batch_leftover_;
public:
- const IterationCount max_iterations;
+ const size_t max_iterations;
private:
bool started_;
@@ -670,14 +667,14 @@
const int threads;
private:
- State(IterationCount max_iters, const std::vector<int64_t>& ranges,
- int thread_i, int n_threads, internal::ThreadTimer* timer,
+ State(size_t max_iters, const std::vector<int64_t>& ranges, int thread_i,
+ int n_threads, internal::ThreadTimer* timer,
internal::ThreadManager* manager);
void StartKeepRunning();
// Implementation of KeepRunning() and KeepRunningBatch().
// is_batch must be true unless n is 1.
- bool KeepRunningInternal(IterationCount n, bool is_batch);
+ bool KeepRunningInternal(size_t n, bool is_batch);
void FinishKeepRunning();
internal::ThreadTimer* timer_;
internal::ThreadManager* manager_;
@@ -689,11 +686,11 @@
return KeepRunningInternal(1, /*is_batch=*/false);
}
-inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(IterationCount n) {
+inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningBatch(size_t n) {
return KeepRunningInternal(n, /*is_batch=*/true);
}
-inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(IterationCount n,
+inline BENCHMARK_ALWAYS_INLINE bool State::KeepRunningInternal(size_t n,
bool is_batch) {
// total_iterations_ is set to 0 by the constructor, and always set to a
// nonzero value by StartKepRunning().
@@ -757,7 +754,7 @@
}
private:
- IterationCount cached_;
+ size_t cached_;
State* const parent_;
};
@@ -861,7 +858,7 @@
// NOTE: This function should only be used when *exact* iteration control is
// needed and never to control or limit how long a benchmark runs, where
// `--benchmark_min_time=N` or `MinTime(...)` should be used instead.
- Benchmark* Iterations(IterationCount n);
+ Benchmark* Iterations(size_t n);
// Specify the amount of times to repeat this benchmark. This option overrides
// the `benchmark_repetitions` flag.
@@ -877,18 +874,11 @@
// Same as ReportAggregatesOnly(), but applies to display reporter only.
Benchmark* DisplayAggregatesOnly(bool value = true);
- // By default, the CPU time is measured only for the main thread, which may
- // be unrepresentative if the benchmark uses threads internally. If called,
- // the total CPU time spent by all the threads will be measured instead.
- // By default, the only the main thread CPU time will be measured.
- Benchmark* MeasureProcessCPUTime();
-
- // If a particular benchmark should use the Wall clock instead of the CPU time
- // (be it either the CPU time of the main thread only (default), or the
- // total CPU usage of the benchmark), call this method. If called, the elapsed
- // (wall) time will be used to control how many iterations are run, and in the
- // printing of items/second or MB/seconds values.
- // If not called, the CPU time used by the benchmark will be used.
+ // If a particular benchmark is I/O bound, runs multiple threads internally or
+ // if for some reason CPU timings are not representative, call this method. If
+ // called, the elapsed time will be used to control how many iterations are
+ // run, and in the printing of items/second or MB/seconds values. If not
+ // called, the cpu time used by the benchmark will be used.
Benchmark* UseRealTime();
// If a benchmark must measure time manually (e.g. if GPU execution time is
@@ -960,9 +950,8 @@
TimeUnit time_unit_;
int range_multiplier_;
double min_time_;
- IterationCount iterations_;
+ size_t iterations_;
int repetitions_;
- bool measure_process_cpu_time_;
bool use_real_time_;
bool use_manual_time_;
BigO complexity_;
@@ -1304,33 +1293,15 @@
BENCHMARK_DISALLOW_COPY_AND_ASSIGN(CPUInfo);
};
-// Adding Struct for System Information
+//Adding Struct for System Information
struct SystemInfo {
std::string name;
static const SystemInfo& Get();
-
private:
SystemInfo();
BENCHMARK_DISALLOW_COPY_AND_ASSIGN(SystemInfo);
};
-// BenchmarkName contains the components of the Benchmark's name
-// which allows individual fields to be modified or cleared before
-// building the final name using 'str()'.
-struct BenchmarkName {
- std::string function_name;
- std::string args;
- std::string min_time;
- std::string iterations;
- std::string repetitions;
- std::string time_type;
- std::string threads;
-
- // Return the full name of the benchmark with each non-empty
- // field separated by a '/'
- std::string str() const;
-};
-
// Interface for custom benchmark result printers.
// By default, benchmark reports are printed to stdout. However an application
// can control the destination of the reports by calling
@@ -1348,14 +1319,12 @@
};
struct Run {
- static const int64_t no_repetition_index = -1;
enum RunType { RT_Iteration, RT_Aggregate };
Run()
: run_type(RT_Iteration),
error_occurred(false),
iterations(1),
- threads(1),
time_unit(kNanosecond),
real_accumulated_time(0),
cpu_accumulated_time(0),
@@ -1371,17 +1340,14 @@
max_bytes_used(0) {}
std::string benchmark_name() const;
- BenchmarkName run_name;
- RunType run_type;
+ std::string run_name;
+ RunType run_type; // is this a measurement, or an aggregate?
std::string aggregate_name;
std::string report_label; // Empty if not set by benchmark.
bool error_occurred;
std::string error_message;
- IterationCount iterations;
- int64_t threads;
- int64_t repetition_index;
- int64_t repetitions;
+ int64_t iterations;
TimeUnit time_unit;
double real_accumulated_time;
double cpu_accumulated_time;
@@ -1519,9 +1485,8 @@
bool first_report_;
};
-class BENCHMARK_DEPRECATED_MSG(
- "The CSV Reporter will be removed in a future release") CSVReporter
- : public BenchmarkReporter {
+class BENCHMARK_DEPRECATED_MSG("The CSV Reporter will be removed in a future release")
+ CSVReporter : public BenchmarkReporter {
public:
CSVReporter() : printed_header_(false) {}
virtual bool ReportContext(const Context& context);
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index eab1428..977474f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,5 +1,4 @@
# Allow the source files to find headers in src/
-include(GNUInstallDirs)
include_directories(${PROJECT_SOURCE_DIR}/src)
if (DEFINED BENCHMARK_CXX_LINKER_FLAGS)
@@ -34,17 +33,9 @@
target_link_libraries(benchmark ${LIBRT})
endif()
-if(CMAKE_BUILD_TYPE)
- string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER)
-endif()
-if(NOT CMAKE_THREAD_LIBS_INIT AND "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}" MATCHES ".*-fsanitize=[^ ]*address.*")
- message(WARNING "CMake's FindThreads.cmake did not fail, but CMAKE_THREAD_LIBS_INIT ended up being empty. This was fixed in https://github.com/Kitware/CMake/commit/d53317130e84898c5328c237186dbd995aaf1c12 Let's guess that -pthread is sufficient.")
- target_link_libraries(benchmark -pthread)
-endif()
-
# We need extra libraries on Windows
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
- target_link_libraries(benchmark shlwapi)
+ target_link_libraries(benchmark Shlwapi)
endif()
# We need extra libraries on Solaris
@@ -64,6 +55,11 @@
)
target_link_libraries(benchmark_main benchmark)
+set(include_install_dir "include")
+set(lib_install_dir "lib/")
+set(bin_install_dir "bin/")
+set(config_install_dir "lib/cmake/${PROJECT_NAME}")
+set(pkgconfig_install_dir "lib/pkgconfig")
set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated")
@@ -87,26 +83,26 @@
install(
TARGETS benchmark benchmark_main
EXPORT ${targets_export_name}
- ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
- LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
- RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
- INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+ ARCHIVE DESTINATION ${lib_install_dir}
+ LIBRARY DESTINATION ${lib_install_dir}
+ RUNTIME DESTINATION ${bin_install_dir}
+ INCLUDES DESTINATION ${include_install_dir})
install(
DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark"
- DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+ DESTINATION ${include_install_dir}
FILES_MATCHING PATTERN "*.*h")
install(
FILES "${project_config}" "${version_config}"
- DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
+ DESTINATION "${config_install_dir}")
install(
FILES "${pkg_config}"
- DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+ DESTINATION "${pkgconfig_install_dir}")
install(
EXPORT "${targets_export_name}"
NAMESPACE "${namespace}"
- DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
+ DESTINATION "${config_install_dir}")
endif()
diff --git a/src/benchmark.cc b/src/benchmark.cc
index 07942eb..aab0750 100644
--- a/src/benchmark.cc
+++ b/src/benchmark.cc
@@ -51,60 +51,66 @@
#include "thread_manager.h"
#include "thread_timer.h"
-// Print a list of benchmarks. This option overrides all other options.
-DEFINE_bool(benchmark_list_tests, false);
+DEFINE_bool(benchmark_list_tests, false,
+ "Print a list of benchmarks. This option overrides all other "
+ "options.");
-// A regular expression that specifies the set of benchmarks to execute. If
-// this flag is empty, or if this flag is the string \"all\", all benchmarks
-// linked into the binary are run.
-DEFINE_string(benchmark_filter, ".");
+DEFINE_string(benchmark_filter, ".",
+ "A regular expression that specifies the set of benchmarks "
+ "to execute. If this flag is empty, or if this flag is the "
+ "string \"all\", all benchmarks linked into the binary are "
+ "run.");
-// Minimum number of seconds we should run benchmark before results are
-// considered significant. For cpu-time based tests, this is the lower bound
-// on the total cpu time used by all threads that make up the test. For
-// real-time based tests, this is the lower bound on the elapsed time of the
-// benchmark execution, regardless of number of threads.
-DEFINE_double(benchmark_min_time, 0.5);
+DEFINE_double(benchmark_min_time, 0.5,
+ "Minimum number of seconds we should run benchmark before "
+ "results are considered significant. For cpu-time based "
+ "tests, this is the lower bound on the total cpu time "
+ "used by all threads that make up the test. For real-time "
+ "based tests, this is the lower bound on the elapsed time "
+ "of the benchmark execution, regardless of number of "
+ "threads.");
-// The number of runs of each benchmark. If greater than 1, the mean and
-// standard deviation of the runs will be reported.
-DEFINE_int32(benchmark_repetitions, 1);
+DEFINE_int32(benchmark_repetitions, 1,
+ "The number of runs of each benchmark. If greater than 1, the "
+ "mean and standard deviation of the runs will be reported.");
-// Report the result of each benchmark repetitions. When 'true' is specified
-// only the mean, standard deviation, and other statistics are reported for
-// repeated benchmarks. Affects all reporters.
-DEFINE_bool(benchmark_report_aggregates_only, false);
+DEFINE_bool(
+ benchmark_report_aggregates_only, false,
+ "Report the result of each benchmark repetitions. When 'true' is specified "
+ "only the mean, standard deviation, and other statistics are reported for "
+ "repeated benchmarks. Affects all reporters.");
-// Display the result of each benchmark repetitions. When 'true' is specified
-// only the mean, standard deviation, and other statistics are displayed for
-// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects
-// the display reporter, but *NOT* file reporter, which will still contain
-// all the output.
-DEFINE_bool(benchmark_display_aggregates_only, false);
+DEFINE_bool(
+ benchmark_display_aggregates_only, false,
+ "Display the result of each benchmark repetitions. When 'true' is "
+ "specified only the mean, standard deviation, and other statistics are "
+ "displayed for repeated benchmarks. Unlike "
+ "benchmark_report_aggregates_only, only affects the display reporter, but "
+ "*NOT* file reporter, which will still contain all the output.");
-// The format to use for console output.
-// Valid values are 'console', 'json', or 'csv'.
-DEFINE_string(benchmark_format, "console");
+DEFINE_string(benchmark_format, "console",
+ "The format to use for console output. Valid values are "
+ "'console', 'json', or 'csv'.");
-// The format to use for file output.
-// Valid values are 'console', 'json', or 'csv'.
-DEFINE_string(benchmark_out_format, "json");
+DEFINE_string(benchmark_out_format, "json",
+ "The format to use for file output. Valid values are "
+ "'console', 'json', or 'csv'.");
-// The file to write additional output to.
-DEFINE_string(benchmark_out, "");
+DEFINE_string(benchmark_out, "", "The file to write additional output to");
-// Whether to use colors in the output. Valid values:
-// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if
-// the output is being sent to a terminal and the TERM environment variable is
-// set to a terminal type that supports colors.
-DEFINE_string(benchmark_color, "auto");
+DEFINE_string(benchmark_color, "auto",
+ "Whether to use colors in the output. Valid values: "
+ "'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use "
+ "colors if the output is being sent to a terminal and the TERM "
+ "environment variable is set to a terminal type that supports "
+ "colors.");
-// Whether to use tabular format when printing user counters to the console.
-// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false.
-DEFINE_bool(benchmark_counters_tabular, false);
+DEFINE_bool(benchmark_counters_tabular, false,
+ "Whether to use tabular format when printing user counters to "
+ "the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0."
+ "Defaults to false.");
-// The level of verbose logging to output
-DEFINE_int32(v, 0);
+DEFINE_int32(v, 0, "The level of verbose logging to output");
namespace benchmark {
@@ -115,8 +121,8 @@
} // namespace internal
-State::State(IterationCount max_iters, const std::vector<int64_t>& ranges,
- int thread_i, int n_threads, internal::ThreadTimer* timer,
+State::State(size_t max_iters, const std::vector<int64_t>& ranges, int thread_i,
+ int n_threads, internal::ThreadTimer* timer,
internal::ThreadManager* manager)
: total_iterations_(0),
batch_leftover_(0),
@@ -142,7 +148,7 @@
// which must be suppressed.
#if defined(__INTEL_COMPILER)
#pragma warning push
-#pragma warning(disable : 1875)
+#pragma warning(disable:1875)
#elif defined(__GNUC__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Winvalid-offsetof"
@@ -227,7 +233,7 @@
size_t stat_field_width = 0;
for (const BenchmarkInstance& benchmark : benchmarks) {
name_field_width =
- std::max<size_t>(name_field_width, benchmark.name.str().size());
+ std::max<size_t>(name_field_width, benchmark.name.size());
might_have_aggregates |= benchmark.repetitions > 1;
for (const auto& Stat : *benchmark.statistics)
@@ -283,13 +289,6 @@
flushStreams(file_reporter);
}
-// Disable deprecated warnings temporarily because we need to reference
-// CSVReporter but don't want to trigger -Werror=-Wdeprecated
-#ifdef __GNUC__
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated"
-#endif
-
std::unique_ptr<BenchmarkReporter> CreateReporter(
std::string const& name, ConsoleReporter::OutputOptions output_opts) {
typedef std::unique_ptr<BenchmarkReporter> PtrType;
@@ -305,10 +304,6 @@
}
}
-#ifdef __GNUC__
-#pragma GCC diagnostic pop
-#endif
-
} // end namespace
bool IsZero(double n) {
@@ -317,7 +312,7 @@
ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) {
int output_opts = ConsoleReporter::OO_Defaults;
- auto is_benchmark_color = [force_no_color]() -> bool {
+ auto is_benchmark_color = [force_no_color] () -> bool {
if (force_no_color) {
return false;
}
@@ -398,8 +393,7 @@
}
if (FLAGS_benchmark_list_tests) {
- for (auto const& benchmark : benchmarks)
- Out << benchmark.name.str() << "\n";
+ for (auto const& benchmark : benchmarks) Out << benchmark.name << "\n";
} else {
internal::RunBenchmarks(benchmarks, display_reporter, file_reporter);
}
diff --git a/src/benchmark_api_internal.cc b/src/benchmark_api_internal.cc
index d468a25..8d31083 100644
--- a/src/benchmark_api_internal.cc
+++ b/src/benchmark_api_internal.cc
@@ -3,9 +3,9 @@
namespace benchmark {
namespace internal {
-State BenchmarkInstance::Run(IterationCount iters, int thread_id,
- internal::ThreadTimer* timer,
- internal::ThreadManager* manager) const {
+State BenchmarkInstance::Run(
+ size_t iters, int thread_id, internal::ThreadTimer* timer,
+ internal::ThreadManager* manager) const {
State st(iters, arg, thread_id, threads, timer, manager);
benchmark->Run(st);
return st;
diff --git a/src/benchmark_api_internal.h b/src/benchmark_api_internal.h
index 264eff9..0524a85 100644
--- a/src/benchmark_api_internal.h
+++ b/src/benchmark_api_internal.h
@@ -16,13 +16,12 @@
// Information kept per benchmark we may want to run
struct BenchmarkInstance {
- BenchmarkName name;
+ std::string name;
Benchmark* benchmark;
AggregationReportMode aggregation_report_mode;
std::vector<int64_t> arg;
TimeUnit time_unit;
int range_multiplier;
- bool measure_process_cpu_time;
bool use_real_time;
bool use_manual_time;
BigO complexity;
@@ -32,10 +31,10 @@
bool last_benchmark_instance;
int repetitions;
double min_time;
- IterationCount iterations;
+ size_t iterations;
int threads; // Number of concurrent threads to us
- State Run(IterationCount iters, int thread_id, internal::ThreadTimer* timer,
+ State Run(size_t iters, int thread_id, internal::ThreadTimer* timer,
internal::ThreadManager* manager) const;
};
diff --git a/src/benchmark_name.cc b/src/benchmark_name.cc
deleted file mode 100644
index 2a17ebc..0000000
--- a/src/benchmark_name.cc
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright 2015 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <benchmark/benchmark.h>
-
-namespace benchmark {
-
-namespace {
-
-// Compute the total size of a pack of std::strings
-size_t size_impl() { return 0; }
-
-template <typename Head, typename... Tail>
-size_t size_impl(const Head& head, const Tail&... tail) {
- return head.size() + size_impl(tail...);
-}
-
-// Join a pack of std::strings using a delimiter
-// TODO: use absl::StrJoin
-void join_impl(std::string&, char) {}
-
-template <typename Head, typename... Tail>
-void join_impl(std::string& s, const char delimiter, const Head& head,
- const Tail&... tail) {
- if (!s.empty() && !head.empty()) {
- s += delimiter;
- }
-
- s += head;
-
- join_impl(s, delimiter, tail...);
-}
-
-template <typename... Ts>
-std::string join(char delimiter, const Ts&... ts) {
- std::string s;
- s.reserve(sizeof...(Ts) + size_impl(ts...));
- join_impl(s, delimiter, ts...);
- return s;
-}
-} // namespace
-
-std::string BenchmarkName::str() const {
- return join('/', function_name, args, min_time, iterations, repetitions,
- time_type, threads);
-}
-} // namespace benchmark
diff --git a/src/benchmark_register.cc b/src/benchmark_register.cc
index cca39b2..f17f5b2 100644
--- a/src/benchmark_register.cc
+++ b/src/benchmark_register.cc
@@ -34,11 +34,6 @@
#include <sstream>
#include <thread>
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-#include <inttypes.h>
-
#include "benchmark/benchmark.h"
#include "benchmark_api_internal.h"
#include "check.h"
@@ -158,7 +153,7 @@
for (auto const& args : family->args_) {
for (int num_threads : *thread_counts) {
BenchmarkInstance instance;
- instance.name.function_name = family->name_;
+ instance.name = family->name_;
instance.benchmark = family.get();
instance.aggregation_report_mode = family->aggregation_report_mode_;
instance.arg = args;
@@ -167,7 +162,6 @@
instance.min_time = family->min_time_;
instance.iterations = family->iterations_;
instance.repetitions = family->repetitions_;
- instance.measure_process_cpu_time = family->measure_process_cpu_time_;
instance.use_real_time = family->use_real_time_;
instance.use_manual_time = family->use_manual_time_;
instance.complexity = family->complexity_;
@@ -178,57 +172,45 @@
// Add arguments to instance name
size_t arg_i = 0;
for (auto const& arg : args) {
- if (!instance.name.args.empty()) {
- instance.name.args += '/';
- }
+ instance.name += "/";
if (arg_i < family->arg_names_.size()) {
const auto& arg_name = family->arg_names_[arg_i];
if (!arg_name.empty()) {
- instance.name.args += StrFormat("%s:", arg_name.c_str());
+ instance.name +=
+ StrFormat("%s:", family->arg_names_[arg_i].c_str());
}
}
- instance.name.args += StrFormat("%" PRId64, arg);
+ // we know that the args are always non-negative (see 'AddRange()'),
+ // thus print as 'unsigned'. BUT, do a cast due to the 32-bit builds.
+ instance.name += StrFormat("%lu", static_cast<unsigned long>(arg));
++arg_i;
}
if (!IsZero(family->min_time_))
- instance.name.min_time =
- StrFormat("min_time:%0.3f", family->min_time_);
+ instance.name += StrFormat("/min_time:%0.3f", family->min_time_);
if (family->iterations_ != 0) {
- instance.name.iterations =
- StrFormat("iterations:%lu",
+ instance.name +=
+ StrFormat("/iterations:%lu",
static_cast<unsigned long>(family->iterations_));
}
if (family->repetitions_ != 0)
- instance.name.repetitions =
- StrFormat("repeats:%d", family->repetitions_);
-
- if (family->measure_process_cpu_time_) {
- instance.name.time_type = "process_time";
- }
+ instance.name += StrFormat("/repeats:%d", family->repetitions_);
if (family->use_manual_time_) {
- if (!instance.name.time_type.empty()) {
- instance.name.time_type += '/';
- }
- instance.name.time_type += "manual_time";
+ instance.name += "/manual_time";
} else if (family->use_real_time_) {
- if (!instance.name.time_type.empty()) {
- instance.name.time_type += '/';
- }
- instance.name.time_type += "real_time";
+ instance.name += "/real_time";
}
// Add the number of threads used to the name
if (!family->thread_counts_.empty()) {
- instance.name.threads = StrFormat("threads:%d", instance.threads);
+ instance.name += StrFormat("/threads:%d", instance.threads);
}
- const auto full_name = instance.name.str();
- if ((re.Match(full_name) && !isNegativeFilter) ||
- (!re.Match(full_name) && isNegativeFilter)) {
+ if ((re.Match(instance.name) && !isNegativeFilter) ||
+ (!re.Match(instance.name) && isNegativeFilter)) {
instance.last_benchmark_instance = (&args == &family->args_.back());
benchmarks->push_back(std::move(instance));
}
@@ -265,7 +247,6 @@
min_time_(0),
iterations_(0),
repetitions_(0),
- measure_process_cpu_time_(false),
use_real_time_(false),
use_manual_time_(false),
complexity_(oNone),
@@ -347,6 +328,7 @@
Benchmark* Benchmark::DenseRange(int64_t start, int64_t limit, int step) {
CHECK(ArgsCnt() == -1 || ArgsCnt() == 1);
+ CHECK_GE(start, 0);
CHECK_LE(start, limit);
for (int64_t arg = start; arg <= limit; arg += step) {
args_.push_back({arg});
@@ -378,7 +360,7 @@
return this;
}
-Benchmark* Benchmark::Iterations(IterationCount n) {
+Benchmark* Benchmark::Iterations(size_t n) {
CHECK(n > 0);
CHECK(IsZero(min_time_));
iterations_ = n;
@@ -412,12 +394,6 @@
return this;
}
-Benchmark* Benchmark::MeasureProcessCPUTime() {
- // Can be used together with UseRealTime() / UseManualTime().
- measure_process_cpu_time_ = true;
- return this;
-}
-
Benchmark* Benchmark::UseRealTime() {
CHECK(!use_manual_time_)
<< "Cannot set UseRealTime and UseManualTime simultaneously.";
diff --git a/src/benchmark_register.h b/src/benchmark_register.h
index 61377d7..0705e21 100644
--- a/src/benchmark_register.h
+++ b/src/benchmark_register.h
@@ -5,103 +5,29 @@
#include "check.h"
-namespace benchmark {
-namespace internal {
-
-// Append the powers of 'mult' in the closed interval [lo, hi].
-// Returns iterator to the start of the inserted range.
-template <typename T>
-typename std::vector<T>::iterator
-AddPowers(std::vector<T>* dst, T lo, T hi, int mult) {
- CHECK_GE(lo, 0);
- CHECK_GE(hi, lo);
- CHECK_GE(mult, 2);
-
- const size_t start_offset = dst->size();
-
- static const T kmax = std::numeric_limits<T>::max();
-
- // Space out the values in multiples of "mult"
- for (T i = 1; i <= hi; i *= mult) {
- if (i >= lo) {
- dst->push_back(i);
- }
- // Break the loop here since multiplying by
- // 'mult' would move outside of the range of T
- if (i > kmax / mult) break;
- }
-
- return dst->begin() + start_offset;
-}
-
-template <typename T>
-void AddNegatedPowers(std::vector<T>* dst, T lo, T hi, int mult) {
- // We negate lo and hi so we require that they cannot be equal to 'min'.
- CHECK_GT(lo, std::numeric_limits<T>::min());
- CHECK_GT(hi, std::numeric_limits<T>::min());
- CHECK_GE(hi, lo);
- CHECK_LE(hi, 0);
-
- // Add positive powers, then negate and reverse.
- // Casts necessary since small integers get promoted
- // to 'int' when negating.
- const auto lo_complement = static_cast<T>(-lo);
- const auto hi_complement = static_cast<T>(-hi);
-
- const auto it = AddPowers(dst, hi_complement, lo_complement, mult);
-
- std::for_each(it, dst->end(), [](T& t) { t *= -1; });
- std::reverse(it, dst->end());
-}
-
template <typename T>
void AddRange(std::vector<T>* dst, T lo, T hi, int mult) {
- static_assert(std::is_integral<T>::value && std::is_signed<T>::value,
- "Args type must be a signed integer");
-
+ CHECK_GE(lo, 0);
CHECK_GE(hi, lo);
CHECK_GE(mult, 2);
// Add "lo"
dst->push_back(lo);
- // Handle lo == hi as a special case, so we then know
- // lo < hi and so it is safe to add 1 to lo and subtract 1
- // from hi without falling outside of the range of T.
- if (lo == hi) return;
+ static const T kmax = std::numeric_limits<T>::max();
- // Ensure that lo_inner <= hi_inner below.
- if (lo + 1 == hi) {
- dst->push_back(hi);
- return;
+ // Now space out the benchmarks in multiples of "mult"
+ for (T i = 1; i < kmax / mult; i *= mult) {
+ if (i >= hi) break;
+ if (i > lo) {
+ dst->push_back(i);
+ }
}
- // Add all powers of 'mult' in the range [lo+1, hi-1] (inclusive).
- const auto lo_inner = static_cast<T>(lo + 1);
- const auto hi_inner = static_cast<T>(hi - 1);
-
- // Insert negative values
- if (lo_inner < 0) {
- AddNegatedPowers(dst, lo_inner, std::min(hi_inner, T{-1}), mult);
- }
-
- // Treat 0 as a special case (see discussion on #762).
- if (lo <= 0 && hi >= 0) {
- dst->push_back(0);
- }
-
- // Insert positive values
- if (hi_inner > 0) {
- AddPowers(dst, std::max(lo_inner, T{1}), hi_inner, mult);
- }
-
- // Add "hi" (if different from last value).
- if (hi != dst->back()) {
+ // Add "hi" (if different from "lo")
+ if (hi != lo) {
dst->push_back(hi);
}
}
-} // namespace internal
-} // namespace benchmark
-
#endif // BENCHMARK_REGISTER_H
diff --git a/src/benchmark_runner.cc b/src/benchmark_runner.cc
index 0bae6a5..38faeec 100644
--- a/src/benchmark_runner.cc
+++ b/src/benchmark_runner.cc
@@ -59,14 +59,12 @@
namespace {
-static constexpr IterationCount kMaxIterations = 1000000000;
+static const size_t kMaxIterations = 1000000000;
BenchmarkReporter::Run CreateRunReport(
const benchmark::internal::BenchmarkInstance& b,
- const internal::ThreadManager::Result& results,
- IterationCount memory_iterations,
- const MemoryManager::Result& memory_result, double seconds,
- int64_t repetition_index) {
+ const internal::ThreadManager::Result& results, size_t memory_iterations,
+ const MemoryManager::Result& memory_result, double seconds) {
// Create report about this benchmark run.
BenchmarkReporter::Run report;
@@ -77,9 +75,6 @@
// This is the total iterations across all threads.
report.iterations = results.iterations;
report.time_unit = b.time_unit;
- report.threads = b.threads;
- report.repetition_index = repetition_index;
- report.repetitions = b.repetitions;
if (!report.error_occurred) {
if (b.use_manual_time) {
@@ -110,12 +105,9 @@
// Execute one thread of benchmark b for the specified number of iterations.
// Adds the stats collected for the thread into *total.
-void RunInThread(const BenchmarkInstance* b, IterationCount iters,
- int thread_id, ThreadManager* manager) {
- internal::ThreadTimer timer(
- b->measure_process_cpu_time
- ? internal::ThreadTimer::CreateProcessCpuTime()
- : internal::ThreadTimer::Create());
+void RunInThread(const BenchmarkInstance* b, size_t iters, int thread_id,
+ ThreadManager* manager) {
+ internal::ThreadTimer timer;
State st = b->Run(iters, thread_id, &timer, manager);
CHECK(st.iterations() >= st.max_iterations)
<< "Benchmark returned before State::KeepRunning() returned false!";
@@ -158,7 +150,8 @@
}
for (int repetition_num = 0; repetition_num < repeats; repetition_num++) {
- DoOneRepetition(repetition_num);
+ const bool is_the_first_repetition = repetition_num == 0;
+ DoOneRepetition(is_the_first_repetition);
}
// Calculate additional statistics
@@ -188,17 +181,17 @@
std::vector<std::thread> pool;
- IterationCount iters; // preserved between repetitions!
+ size_t iters; // preserved between repetitions!
// So only the first repetition has to find/calculate it,
// the other repetitions will just use that precomputed iteration count.
struct IterationResults {
internal::ThreadManager::Result results;
- IterationCount iters;
+ size_t iters;
double seconds;
};
IterationResults DoNIterations() {
- VLOG(2) << "Running " << b.name.str() << " for " << iters << "\n";
+ VLOG(2) << "Running " << b.name << " for " << iters << "\n";
std::unique_ptr<internal::ThreadManager> manager;
manager.reset(new internal::ThreadManager(b.threads));
@@ -230,8 +223,6 @@
// Adjust real/manual time stats since they were reported per thread.
i.results.real_time_used /= b.threads;
i.results.manual_time_used /= b.threads;
- // If we were measuring whole-process CPU usage, adjust the CPU time too.
- if (b.measure_process_cpu_time) i.results.cpu_time_used /= b.threads;
VLOG(2) << "Ran in " << i.results.cpu_time_used << "/"
<< i.results.real_time_used << "\n";
@@ -249,7 +240,7 @@
return i;
}
- IterationCount PredictNumItersNeeded(const IterationResults& i) const {
+ size_t PredictNumItersNeeded(const IterationResults& i) const {
// See how much iterations should be increased by.
// Note: Avoid division by zero with max(seconds, 1ns).
double multiplier = min_time * 1.4 / std::max(i.seconds, 1e-9);
@@ -263,10 +254,10 @@
if (multiplier <= 1.0) multiplier = 2.0;
// So what seems to be the sufficiently-large iteration count? Round up.
- const IterationCount max_next_iters =
+ const size_t max_next_iters =
0.5 + std::max(multiplier * i.iters, i.iters + 1.0);
// But we do have *some* sanity limits though..
- const IterationCount next_iters = std::min(max_next_iters, kMaxIterations);
+ const size_t next_iters = std::min(max_next_iters, kMaxIterations);
VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n";
return next_iters; // round up before conversion to integer.
@@ -285,8 +276,7 @@
((i.results.real_time_used >= 5 * min_time) && !b.use_manual_time);
}
- void DoOneRepetition(int64_t repetition_index) {
- const bool is_the_first_repetition = repetition_index == 0;
+ void DoOneRepetition(bool is_the_first_repetition) {
IterationResults i;
// We *may* be gradually increasing the length (iteration count)
@@ -320,11 +310,11 @@
// Oh, one last thing, we need to also produce the 'memory measurements'..
MemoryManager::Result memory_result;
- IterationCount memory_iterations = 0;
+ size_t memory_iterations = 0;
if (memory_manager != nullptr) {
// Only run a few iterations to reduce the impact of one-time
// allocations in benchmarks that are not properly managed.
- memory_iterations = std::min<IterationCount>(16, iters);
+ memory_iterations = std::min<size_t>(16, iters);
memory_manager->Start();
std::unique_ptr<internal::ThreadManager> manager;
manager.reset(new internal::ThreadManager(1));
@@ -336,9 +326,8 @@
}
// Ok, now actualy report.
- BenchmarkReporter::Run report =
- CreateRunReport(b, i.results, memory_iterations, memory_result,
- i.seconds, repetition_index);
+ BenchmarkReporter::Run report = CreateRunReport(
+ b, i.results, memory_iterations, memory_result, i.seconds);
if (!report.error_occurred && b.complexity != oNone)
complexity_reports.push_back(report);
diff --git a/src/commandlineflags.cc b/src/commandlineflags.cc
index 4e60f0b..734e88b 100644
--- a/src/commandlineflags.cc
+++ b/src/commandlineflags.cc
@@ -14,7 +14,6 @@
#include "commandlineflags.h"
-#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <cstring>
@@ -22,8 +21,6 @@
#include <limits>
namespace benchmark {
-namespace {
-
// Parses 'str' for a 32-bit signed integer. If successful, writes
// the result to *value and returns true; otherwise leaves *value
// unchanged and returns false.
@@ -91,42 +88,44 @@
return "BENCHMARK_" + env_var;
}
-} // namespace
-
-bool BoolFromEnv(const char* flag, bool default_val) {
+// Reads and returns the Boolean environment variable corresponding to
+// the given flag; if it's not set, returns default_value.
+//
+// The value is considered true iff it's not "0".
+bool BoolFromEnv(const char* flag, bool default_value) {
const std::string env_var = FlagToEnvVar(flag);
- const char* const value_str = getenv(env_var.c_str());
- return value_str == nullptr ? default_val : IsTruthyFlagValue(value_str);
+ const char* const string_value = getenv(env_var.c_str());
+ return string_value == nullptr ? default_value
+ : strcmp(string_value, "0") != 0;
}
-int32_t Int32FromEnv(const char* flag, int32_t default_val) {
+// Reads and returns a 32-bit integer stored in the environment
+// variable corresponding to the given flag; if it isn't set or
+// doesn't represent a valid 32-bit integer, returns default_value.
+int32_t Int32FromEnv(const char* flag, int32_t default_value) {
const std::string env_var = FlagToEnvVar(flag);
- const char* const value_str = getenv(env_var.c_str());
- int32_t value = default_val;
- if (value_str == nullptr ||
- !ParseInt32(std::string("Environment variable ") + env_var, value_str,
- &value)) {
- return default_val;
+ const char* const string_value = getenv(env_var.c_str());
+ if (string_value == nullptr) {
+ // The environment variable is not set.
+ return default_value;
}
- return value;
-}
-double DoubleFromEnv(const char* flag, double default_val) {
- const std::string env_var = FlagToEnvVar(flag);
- const char* const value_str = getenv(env_var.c_str());
- double value = default_val;
- if (value_str == nullptr ||
- !ParseDouble(std::string("Environment variable ") + env_var, value_str,
- &value)) {
- return default_val;
+ int32_t result = default_value;
+ if (!ParseInt32(std::string("Environment variable ") + env_var, string_value,
+ &result)) {
+ std::cout << "The default value " << default_value << " is used.\n";
+ return default_value;
}
- return value;
+
+ return result;
}
-const char* StringFromEnv(const char* flag, const char* default_val) {
+// Reads and returns the string environment variable corresponding to
+// the given flag; if it's not set, returns default_value.
+const char* StringFromEnv(const char* flag, const char* default_value) {
const std::string env_var = FlagToEnvVar(flag);
const char* const value = getenv(env_var.c_str());
- return value == nullptr ? default_val : value;
+ return value == nullptr ? default_value : value;
}
// Parses a string as a command line flag. The string should have
@@ -211,18 +210,9 @@
}
bool IsTruthyFlagValue(const std::string& value) {
- if (value.size() == 1) {
- char v = value[0];
- return isalnum(v) &&
- !(v == '0' || v == 'f' || v == 'F' || v == 'n' || v == 'N');
- } else if (!value.empty()) {
- std::string value_lower(value);
- std::transform(value_lower.begin(), value_lower.end(),
- value_lower.begin(), ::tolower);
- return !(value_lower == "false" || value_lower == "no" ||
- value_lower == "off");
- } else
- return true;
+ if (value.empty()) return true;
+ char ch = value[0];
+ return isalnum(ch) &&
+ !(ch == '0' || ch == 'f' || ch == 'F' || ch == 'n' || ch == 'N');
}
-
} // end namespace benchmark
diff --git a/src/commandlineflags.h b/src/commandlineflags.h
index 3a1f6a8..945c9a9 100644
--- a/src/commandlineflags.h
+++ b/src/commandlineflags.h
@@ -10,51 +10,29 @@
// Macros for declaring flags.
#define DECLARE_bool(name) extern bool FLAG(name)
#define DECLARE_int32(name) extern int32_t FLAG(name)
+#define DECLARE_int64(name) extern int64_t FLAG(name)
#define DECLARE_double(name) extern double FLAG(name)
#define DECLARE_string(name) extern std::string FLAG(name)
// Macros for defining flags.
-#define DEFINE_bool(name, default_val) \
- bool FLAG(name) = \
- benchmark::BoolFromEnv(#name, default_val)
-#define DEFINE_int32(name, default_val) \
- int32_t FLAG(name) = \
- benchmark::Int32FromEnv(#name, default_val)
-#define DEFINE_double(name, default_val) \
- double FLAG(name) = \
- benchmark::DoubleFromEnv(#name, default_val)
-#define DEFINE_string(name, default_val) \
- std::string FLAG(name) = \
- benchmark::StringFromEnv(#name, default_val)
+#define DEFINE_bool(name, default_val, doc) bool FLAG(name) = (default_val)
+#define DEFINE_int32(name, default_val, doc) int32_t FLAG(name) = (default_val)
+#define DEFINE_int64(name, default_val, doc) int64_t FLAG(name) = (default_val)
+#define DEFINE_double(name, default_val, doc) double FLAG(name) = (default_val)
+#define DEFINE_string(name, default_val, doc) \
+ std::string FLAG(name) = (default_val)
namespace benchmark {
+// Parses 'str' for a 32-bit signed integer. If successful, writes the result
+// to *value and returns true; otherwise leaves *value unchanged and returns
+// false.
+bool ParseInt32(const std::string& src_text, const char* str, int32_t* value);
-// Parses a bool from the environment variable
-// corresponding to the given flag.
-//
-// If the variable exists, returns IsTruthyFlagValue() value; if not,
-// returns the given default value.
+// Parses a bool/Int32/string from the environment variable
+// corresponding to the given Google Test flag.
bool BoolFromEnv(const char* flag, bool default_val);
-
-// Parses an Int32 from the environment variable
-// corresponding to the given flag.
-//
-// If the variable exists, returns ParseInt32() value; if not, returns
-// the given default value.
int32_t Int32FromEnv(const char* flag, int32_t default_val);
-
-// Parses an Double from the environment variable
-// corresponding to the given flag.
-//
-// If the variable exists, returns ParseDouble(); if not, returns
-// the given default value.
double DoubleFromEnv(const char* flag, double default_val);
-
-// Parses a string from the environment variable
-// corresponding to the given flag.
-//
-// If variable exists, returns its value; if not, returns
-// the given default value.
const char* StringFromEnv(const char* flag, const char* default_val);
// Parses a string for a bool flag, in the form of either
@@ -93,11 +71,9 @@
bool IsFlag(const char* str, const char* flag);
// Returns true unless value starts with one of: '0', 'f', 'F', 'n' or 'N', or
-// some non-alphanumeric character. Also returns false if the value matches
-// one of 'no', 'false', 'off' (case-insensitive). As a special case, also
-// returns true if value is the empty string.
+// some non-alphanumeric character. As a special case, also returns true if
+// value is the empty string.
bool IsTruthyFlagValue(const std::string& value);
-
} // end namespace benchmark
#endif // BENCHMARK_COMMANDLINEFLAGS_H_
diff --git a/src/complexity.cc b/src/complexity.cc
index aeed67f..6ef1766 100644
--- a/src/complexity.cc
+++ b/src/complexity.cc
@@ -29,23 +29,20 @@
static const double kLog2E = 1.44269504088896340736;
switch (complexity) {
case oN:
- return [](IterationCount n) -> double { return static_cast<double>(n); };
+ return [](int64_t n) -> double { return static_cast<double>(n); };
case oNSquared:
- return [](IterationCount n) -> double { return std::pow(n, 2); };
+ return [](int64_t n) -> double { return std::pow(n, 2); };
case oNCubed:
- return [](IterationCount n) -> double { return std::pow(n, 3); };
+ return [](int64_t n) -> double { return std::pow(n, 3); };
case oLogN:
/* Note: can't use log2 because Android's GNU STL lacks it */
- return
- [](IterationCount n) { return kLog2E * log(static_cast<double>(n)); };
+ return [](int64_t n) { return kLog2E * log(static_cast<double>(n)); };
case oNLogN:
/* Note: can't use log2 because Android's GNU STL lacks it */
- return [](IterationCount n) {
- return kLog2E * n * log(static_cast<double>(n));
- };
+ return [](int64_t n) { return kLog2E * n * log(static_cast<double>(n)); };
case o1:
default:
- return [](IterationCount) { return 1.0; };
+ return [](int64_t) { return 1.0; };
}
}
@@ -186,19 +183,14 @@
result_real = MinimalLeastSq(n, real_time, result_cpu.complexity);
}
- // Drop the 'args' when reporting complexity.
- auto run_name = reports[0].run_name;
- run_name.args.clear();
+ std::string run_name = reports[0].benchmark_name().substr(
+ 0, reports[0].benchmark_name().find('/'));
// Get the data from the accumulator to BenchmarkReporter::Run's.
Run big_o;
big_o.run_name = run_name;
big_o.run_type = BenchmarkReporter::Run::RT_Aggregate;
- big_o.repetitions = reports[0].repetitions;
- big_o.repetition_index = Run::no_repetition_index;
- big_o.threads = reports[0].threads;
big_o.aggregate_name = "BigO";
- big_o.report_label = reports[0].report_label;
big_o.iterations = 0;
big_o.real_accumulated_time = result_real.coef;
big_o.cpu_accumulated_time = result_cpu.coef;
@@ -215,13 +207,11 @@
// Only add label to mean/stddev if it is same for all runs
Run rms;
rms.run_name = run_name;
+ big_o.report_label = reports[0].report_label;
rms.run_type = BenchmarkReporter::Run::RT_Aggregate;
rms.aggregate_name = "RMS";
rms.report_label = big_o.report_label;
rms.iterations = 0;
- rms.repetition_index = Run::no_repetition_index;
- rms.repetitions = reports[0].repetitions;
- rms.threads = reports[0].threads;
rms.real_accumulated_time = result_real.rms / multiplier;
rms.cpu_accumulated_time = result_cpu.rms / multiplier;
rms.report_rms = true;
diff --git a/src/console_reporter.cc b/src/console_reporter.cc
index 6fd7645..cc8ae27 100644
--- a/src/console_reporter.cc
+++ b/src/console_reporter.cc
@@ -12,21 +12,21 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "benchmark/benchmark.h"
+#include "complexity.h"
+#include "counter.h"
+
#include <algorithm>
#include <cstdint>
#include <cstdio>
-#include <cstring>
#include <iostream>
#include <string>
#include <tuple>
#include <vector>
-#include "benchmark/benchmark.h"
#include "check.h"
#include "colorprint.h"
#include "commandlineflags.h"
-#include "complexity.h"
-#include "counter.h"
#include "internal_macros.h"
#include "string_util.h"
#include "timers.h"
@@ -156,14 +156,16 @@
const std::size_t cNameLen = std::max(std::string::size_type(10),
c.first.length());
auto const& s = HumanReadableNumber(c.second.value, c.second.oneK);
- const char* unit = "";
- if (c.second.flags & Counter::kIsRate)
- unit = (c.second.flags & Counter::kInvert) ? "s" : "/s";
if (output_options_ & OO_Tabular) {
- printer(Out, COLOR_DEFAULT, " %*s%s", cNameLen - strlen(unit), s.c_str(),
- unit);
+ if (c.second.flags & Counter::kIsRate) {
+ printer(Out, COLOR_DEFAULT, " %*s/s", cNameLen - 2, s.c_str());
+ } else {
+ printer(Out, COLOR_DEFAULT, " %*s", cNameLen, s.c_str());
+ }
} else {
- printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), unit);
+ const char* unit = (c.second.flags & Counter::kIsRate) ? "/s" : "";
+ printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(),
+ unit);
}
}
diff --git a/src/counter.cc b/src/counter.cc
index cf5b78e..cb604e0 100644
--- a/src/counter.cc
+++ b/src/counter.cc
@@ -17,7 +17,7 @@
namespace benchmark {
namespace internal {
-double Finish(Counter const& c, IterationCount iterations, double cpu_time,
+double Finish(Counter const& c, int64_t iterations, double cpu_time,
double num_threads) {
double v = c.value;
if (c.flags & Counter::kIsRate) {
@@ -32,15 +32,10 @@
if (c.flags & Counter::kAvgIterations) {
v /= iterations;
}
-
- if (c.flags & Counter::kInvert) { // Invert is *always* last.
- v = 1.0 / v;
- }
return v;
}
-void Finish(UserCounters* l, IterationCount iterations, double cpu_time,
- double num_threads) {
+void Finish(UserCounters* l, int64_t iterations, double cpu_time, double num_threads) {
for (auto& c : *l) {
c.second.value = Finish(c.second, iterations, cpu_time, num_threads);
}
diff --git a/src/counter.h b/src/counter.h
index 1ad46d4..d884e50 100644
--- a/src/counter.h
+++ b/src/counter.h
@@ -18,8 +18,7 @@
// these counter-related functions are hidden to reduce API surface.
namespace internal {
-void Finish(UserCounters* l, IterationCount iterations, double time,
- double num_threads);
+void Finish(UserCounters* l, int64_t iterations, double time, double num_threads);
void Increment(UserCounters* l, UserCounters const& r);
bool SameNames(UserCounters const& l, UserCounters const& r);
} // end namespace internal
diff --git a/src/csv_reporter.cc b/src/csv_reporter.cc
index af2c18f..d2f1d27 100644
--- a/src/csv_reporter.cc
+++ b/src/csv_reporter.cc
@@ -37,18 +37,6 @@
"error_occurred", "error_message"};
} // namespace
-std::string CsvEscape(const std::string & s) {
- std::string tmp;
- tmp.reserve(s.size() + 2);
- for (char c : s) {
- switch (c) {
- case '"' : tmp += "\"\""; break;
- default : tmp += c; break;
- }
- }
- return '"' + tmp + '"';
-}
-
bool CSVReporter::ReportContext(const Context& context) {
PrintBasicContext(&GetErrorStream(), context);
return true;
@@ -101,11 +89,18 @@
void CSVReporter::PrintRunData(const Run& run) {
std::ostream& Out = GetOutputStream();
- Out << CsvEscape(run.benchmark_name()) << ",";
+
+ // Field with embedded double-quote characters must be doubled and the field
+ // delimited with double-quotes.
+ std::string name = run.benchmark_name();
+ ReplaceAll(&name, "\"", "\"\"");
+ Out << '"' << name << "\",";
if (run.error_occurred) {
Out << std::string(elements.size() - 3, ',');
Out << "true,";
- Out << CsvEscape(run.error_message) << "\n";
+ std::string msg = run.error_message;
+ ReplaceAll(&msg, "\"", "\"\"");
+ Out << '"' << msg << "\"\n";
return;
}
@@ -135,7 +130,11 @@
}
Out << ",";
if (!run.report_label.empty()) {
- Out << CsvEscape(run.report_label);
+ // Field with embedded double-quote characters must be doubled and the field
+ // delimited with double-quotes.
+ std::string label = run.report_label;
+ ReplaceAll(&label, "\"", "\"\"");
+ Out << "\"" << label << "\"";
}
Out << ",,"; // for error_occurred and error_message
diff --git a/src/cycleclock.h b/src/cycleclock.h
index d5d62c4..f5e37b0 100644
--- a/src/cycleclock.h
+++ b/src/cycleclock.h
@@ -164,21 +164,6 @@
uint64_t tsc;
asm("stck %0" : "=Q"(tsc) : : "cc");
return tsc;
-#elif defined(__riscv) // RISC-V
- // Use RDCYCLE (and RDCYCLEH on riscv32)
-#if __riscv_xlen == 32
- uint64_t cycles_low, cycles_hi0, cycles_hi1;
- asm("rdcycleh %0" : "=r"(cycles_hi0));
- asm("rdcycle %0" : "=r"(cycles_lo));
- asm("rdcycleh %0" : "=r"(cycles_hi1));
- // This matches the PowerPC overflow detection, above
- cycles_lo &= -static_cast<int64_t>(cycles_hi0 == cycles_hi1);
- return (cycles_hi1 << 32) | cycles_lo;
-#else
- uint64_t cycles;
- asm("rdcycle %0" : "=r"(cycles));
- return cycles;
-#endif
#else
// The soft failover to a generic implementation is automatic only for ARM.
// For other platforms the developer is expected to make an attempt to create
diff --git a/src/json_reporter.cc b/src/json_reporter.cc
index fe7b1be..7d01e8e 100644
--- a/src/json_reporter.cc
+++ b/src/json_reporter.cc
@@ -16,7 +16,6 @@
#include "complexity.h"
#include <algorithm>
-#include <cmath>
#include <cstdint>
#include <iomanip> // for setprecision
#include <iostream>
@@ -32,63 +31,32 @@
namespace {
-std::string StrEscape(const std::string & s) {
- std::string tmp;
- tmp.reserve(s.size());
- for (char c : s) {
- switch (c) {
- case '\b': tmp += "\\b"; break;
- case '\f': tmp += "\\f"; break;
- case '\n': tmp += "\\n"; break;
- case '\r': tmp += "\\r"; break;
- case '\t': tmp += "\\t"; break;
- case '\\': tmp += "\\\\"; break;
- case '"' : tmp += "\\\""; break;
- default : tmp += c; break;
- }
- }
- return tmp;
-}
-
std::string FormatKV(std::string const& key, std::string const& value) {
- return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str());
+ return StrFormat("\"%s\": \"%s\"", key.c_str(), value.c_str());
}
std::string FormatKV(std::string const& key, const char* value) {
- return StrFormat("\"%s\": \"%s\"", StrEscape(key).c_str(), StrEscape(value).c_str());
+ return StrFormat("\"%s\": \"%s\"", key.c_str(), value);
}
std::string FormatKV(std::string const& key, bool value) {
- return StrFormat("\"%s\": %s", StrEscape(key).c_str(), value ? "true" : "false");
+ return StrFormat("\"%s\": %s", key.c_str(), value ? "true" : "false");
}
std::string FormatKV(std::string const& key, int64_t value) {
std::stringstream ss;
- ss << '"' << StrEscape(key) << "\": " << value;
- return ss.str();
-}
-
-std::string FormatKV(std::string const& key, IterationCount value) {
- std::stringstream ss;
- ss << '"' << StrEscape(key) << "\": " << value;
+ ss << '"' << key << "\": " << value;
return ss.str();
}
std::string FormatKV(std::string const& key, double value) {
std::stringstream ss;
- ss << '"' << StrEscape(key) << "\": ";
+ ss << '"' << key << "\": ";
- if (std::isnan(value))
- ss << (value < 0 ? "-" : "") << "NaN";
- else if (std::isinf(value))
- ss << (value < 0 ? "-" : "") << "Infinity";
- else {
- const auto max_digits10 =
- std::numeric_limits<decltype(value)>::max_digits10;
- const auto max_fractional_digits10 = max_digits10 - 1;
- ss << std::scientific << std::setprecision(max_fractional_digits10)
- << value;
- }
+ const auto max_digits10 = std::numeric_limits<decltype(value)>::max_digits10;
+ const auto max_fractional_digits10 = max_digits10 - 1;
+
+ ss << std::scientific << std::setprecision(max_fractional_digits10) << value;
return ss.str();
}
@@ -112,7 +80,12 @@
out << indent << FormatKV("host_name", context.sys_info.name) << ",\n";
if (Context::executable_name) {
- out << indent << FormatKV("executable", Context::executable_name) << ",\n";
+ // windows uses backslash for its path separator,
+ // which must be escaped in JSON otherwise it blows up conforming JSON
+ // decoders
+ std::string executable_name = Context::executable_name;
+ ReplaceAll(&executable_name, "\\", "\\\\");
+ out << indent << FormatKV("executable", executable_name) << ",\n";
}
CPUInfo const& info = context.cpu_info;
@@ -135,7 +108,7 @@
out << cache_indent << FormatKV("level", static_cast<int64_t>(CI.level))
<< ",\n";
out << cache_indent
- << FormatKV("size", static_cast<int64_t>(CI.size)) << ",\n";
+ << FormatKV("size", static_cast<int64_t>(CI.size) * 1000u) << ",\n";
out << cache_indent
<< FormatKV("num_sharing", static_cast<int64_t>(CI.num_sharing))
<< "\n";
@@ -195,7 +168,7 @@
std::string indent(6, ' ');
std::ostream& out = GetOutputStream();
out << indent << FormatKV("name", run.benchmark_name()) << ",\n";
- out << indent << FormatKV("run_name", run.run_name.str()) << ",\n";
+ out << indent << FormatKV("run_name", run.run_name) << ",\n";
out << indent << FormatKV("run_type", [&run]() -> const char* {
switch (run.run_type) {
case BenchmarkReporter::Run::RT_Iteration:
@@ -205,12 +178,6 @@
}
BENCHMARK_UNREACHABLE();
}()) << ",\n";
- out << indent << FormatKV("repetitions", run.repetitions) << ",\n";
- if (run.run_type != BenchmarkReporter::Run::RT_Aggregate) {
- out << indent << FormatKV("repetition_index", run.repetition_index)
- << ",\n";
- }
- out << indent << FormatKV("threads", run.threads) << ",\n";
if (run.run_type == BenchmarkReporter::Run::RT_Aggregate) {
out << indent << FormatKV("aggregate_name", run.aggregate_name) << ",\n";
}
diff --git a/src/reporter.cc b/src/reporter.cc
index 4d3e477..59bc5f7 100644
--- a/src/reporter.cc
+++ b/src/reporter.cc
@@ -83,7 +83,7 @@
: cpu_info(CPUInfo::Get()), sys_info(SystemInfo::Get()) {}
std::string BenchmarkReporter::Run::benchmark_name() const {
- std::string name = run_name.str();
+ std::string name = run_name;
if (run_type == RT_Aggregate) {
name += "_" + aggregate_name;
}
diff --git a/src/statistics.cc b/src/statistics.cc
index bd5a3d6..e821aec 100644
--- a/src/statistics.cc
+++ b/src/statistics.cc
@@ -97,7 +97,7 @@
// All repetitions should be run with the same number of iterations so we
// can take this information from the first benchmark.
- const IterationCount run_iterations = reports.front().iterations;
+ int64_t const run_iterations = reports.front().iterations;
// create stats for user counters
struct CounterStat {
Counter c;
@@ -147,11 +147,8 @@
for (const auto& Stat : *reports[0].statistics) {
// Get the data from the accumulator to BenchmarkReporter::Run's.
Run data;
- data.run_name = reports[0].run_name;
+ data.run_name = reports[0].benchmark_name();
data.run_type = BenchmarkReporter::Run::RT_Aggregate;
- data.threads = reports[0].threads;
- data.repetitions = reports[0].repetitions;
- data.repetition_index = Run::no_repetition_index;
data.aggregate_name = Stat.name_;
data.report_label = report_label;
diff --git a/src/string_util.cc b/src/string_util.cc
index 39b01a1..05ac5b4 100644
--- a/src/string_util.cc
+++ b/src/string_util.cc
@@ -160,6 +160,15 @@
return tmp;
}
+void ReplaceAll(std::string* str, const std::string& from,
+ const std::string& to) {
+ std::size_t start = 0;
+ while ((start = str->find(from, start)) != std::string::npos) {
+ str->replace(start, from.length(), to);
+ start += to.length();
+ }
+}
+
#ifdef BENCHMARK_STL_ANDROID_GNUSTL
/*
* GNU STL in Android NDK lacks support for some C++11 functions, including
diff --git a/src/string_util.h b/src/string_util.h
index 09d7b4b..fc5f8b0 100644
--- a/src/string_util.h
+++ b/src/string_util.h
@@ -12,9 +12,7 @@
std::string HumanReadableNumber(double n, double one_k = 1024.0);
-#if defined(__MINGW32__)
-__attribute__((format(__MINGW_PRINTF_FORMAT, 1, 2)))
-#elif defined(__GNUC__)
+#ifdef __GNUC__
__attribute__((format(printf, 1, 2)))
#endif
std::string
@@ -37,6 +35,9 @@
return ss.str();
}
+void ReplaceAll(std::string* str, const std::string& from,
+ const std::string& to);
+
#ifdef BENCHMARK_STL_ANDROID_GNUSTL
/*
* GNU STL in Android NDK lacks support for some C++11 functions, including
diff --git a/src/sysinfo.cc b/src/sysinfo.cc
index b5f99c7..953c170 100644
--- a/src/sysinfo.cc
+++ b/src/sysinfo.cc
@@ -429,20 +429,11 @@
#endif
return str;
#else // defined(BENCHMARK_OS_WINDOWS)
-#ifndef HOST_NAME_MAX
-#ifdef BENCHMARK_HAS_SYSCTL // BSD/Mac Doesnt have HOST_NAME_MAX defined
-#define HOST_NAME_MAX 64
-#elif defined(BENCHMARK_OS_NACL)
+#ifdef BENCHMARK_OS_MACOSX //Mac Doesnt have HOST_NAME_MAX defined
#define HOST_NAME_MAX 64
#elif defined(BENCHMARK_OS_QNX)
#define HOST_NAME_MAX 154
-#elif defined(BENCHMARK_OS_RTEMS)
-#define HOST_NAME_MAX 256
-#else
-#warning "HOST_NAME_MAX not defined. using 64"
-#define HOST_NAME_MAX 64
#endif
-#endif // def HOST_NAME_MAX
char hostname[HOST_NAME_MAX];
int retVal = gethostname(hostname, HOST_NAME_MAX);
if (retVal != 0) return std::string("");
@@ -667,9 +658,9 @@
}
std::vector<double> GetLoadAvg() {
-#if (defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \
+#if defined BENCHMARK_OS_FREEBSD || defined(BENCHMARK_OS_LINUX) || \
defined BENCHMARK_OS_MACOSX || defined BENCHMARK_OS_NETBSD || \
- defined BENCHMARK_OS_OPENBSD) && !defined(__ANDROID__)
+ defined BENCHMARK_OS_OPENBSD
constexpr int kMaxSamples = 3;
std::vector<double> res(kMaxSamples, 0.0);
const int nelem = getloadavg(res.data(), kMaxSamples);
diff --git a/src/thread_manager.h b/src/thread_manager.h
index 28e2dd5..6e274c7 100644
--- a/src/thread_manager.h
+++ b/src/thread_manager.h
@@ -11,7 +11,7 @@
class ThreadManager {
public:
- explicit ThreadManager(int num_threads)
+ ThreadManager(int num_threads)
: alive_threads_(num_threads), start_stop_barrier_(num_threads) {}
Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) {
@@ -38,7 +38,7 @@
public:
struct Result {
- IterationCount iterations = 0;
+ int64_t iterations = 0;
double real_time_used = 0;
double cpu_time_used = 0;
double manual_time_used = 0;
diff --git a/src/thread_timer.h b/src/thread_timer.h
index 1703ca0..eaf108e 100644
--- a/src/thread_timer.h
+++ b/src/thread_timer.h
@@ -8,22 +8,14 @@
namespace internal {
class ThreadTimer {
- explicit ThreadTimer(bool measure_process_cpu_time_)
- : measure_process_cpu_time(measure_process_cpu_time_) {}
-
public:
- static ThreadTimer Create() {
- return ThreadTimer(/*measure_process_cpu_time_=*/false);
- }
- static ThreadTimer CreateProcessCpuTime() {
- return ThreadTimer(/*measure_process_cpu_time_=*/true);
- }
+ ThreadTimer() = default;
// Called by each thread
void StartTimer() {
running_ = true;
start_real_time_ = ChronoClockNow();
- start_cpu_time_ = ReadCpuTimerOfChoice();
+ start_cpu_time_ = ThreadCPUUsage();
}
// Called by each thread
@@ -33,8 +25,7 @@
real_time_used_ += ChronoClockNow() - start_real_time_;
// Floating point error can result in the subtraction producing a negative
// time. Guard against that.
- cpu_time_used_ +=
- std::max<double>(ReadCpuTimerOfChoice() - start_cpu_time_, 0);
+ cpu_time_used_ += std::max<double>(ThreadCPUUsage() - start_cpu_time_, 0);
}
// Called by each thread
@@ -43,32 +34,24 @@
bool running() const { return running_; }
// REQUIRES: timer is not running
- double real_time_used() const {
+ double real_time_used() {
CHECK(!running_);
return real_time_used_;
}
// REQUIRES: timer is not running
- double cpu_time_used() const {
+ double cpu_time_used() {
CHECK(!running_);
return cpu_time_used_;
}
// REQUIRES: timer is not running
- double manual_time_used() const {
+ double manual_time_used() {
CHECK(!running_);
return manual_time_used_;
}
private:
- double ReadCpuTimerOfChoice() const {
- if (measure_process_cpu_time) return ProcessCPUUsage();
- return ThreadCPUUsage();
- }
-
- // should the thread, or the process, time be measured?
- const bool measure_process_cpu_time;
-
bool running_ = false; // Is the timer running
double start_real_time_ = 0; // If running_
double start_cpu_time_ = 0; // If running_
diff --git a/test/BUILD b/test/BUILD
index 9bb8cb0..3f174c4 100644
--- a/test/BUILD
+++ b/test/BUILD
@@ -5,7 +5,7 @@
"-Wall",
"-Wextra",
"-Wshadow",
- # "-Wshorten-64-to-32",
+# "-Wshorten-64-to-32",
"-Wfloat-equal",
"-fstrict-aliasing",
]
@@ -16,14 +16,13 @@
"donotoptimize_test.cc": ["-O3"],
})
+
TEST_ARGS = ["--benchmark_min_time=0.01"]
PER_SRC_TEST_ARGS = ({
"user_counters_tabular_test.cc": ["--benchmark_counters_tabular=true"],
})
-load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
-
cc_library(
name = "output_test_helper",
testonly = 1,
@@ -37,31 +36,24 @@
)
[
- cc_test(
- name = test_src[:-len(".cc")],
- size = "small",
- srcs = [test_src],
- args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []),
- copts = TEST_COPTS + PER_SRC_COPTS.get(test_src, []),
- deps = [
- ":output_test_helper",
- "//:benchmark",
- "//:benchmark_internal_headers",
- "@com_google_googletest//:gtest",
- ] + (
- ["@com_google_googletest//:gtest_main"] if (test_src[-len("gtest.cc"):] == "gtest.cc") else []
- ),
- # FIXME: Add support for assembly tests to bazel.
- # See Issue #556
- # https://github.com/google/benchmark/issues/556
- )
- for test_src in glob(
- ["*test.cc"],
- exclude = [
- "*_assembly_test.cc",
- "link_main_test.cc",
- ],
- )
+ cc_test(
+ name = test_src[:-len(".cc")],
+ size = "small",
+ srcs = [test_src],
+ args = TEST_ARGS + PER_SRC_TEST_ARGS.get(test_src, []),
+ copts = TEST_COPTS + PER_SRC_COPTS.get(test_src, []),
+ deps = [
+ ":output_test_helper",
+ "//:benchmark",
+ "//:benchmark_internal_headers",
+ "@com_google_googletest//:gtest",
+ ] + (
+ ["@com_google_googletest//:gtest_main"] if (test_src[-len("gtest.cc"):] == "gtest.cc") else []
+ ),
+ # FIXME: Add support for assembly tests to bazel.
+ # See Issue #556
+ # https://github.com/google/benchmark/issues/556
+ ) for test_src in glob(["*test.cc"], exclude = ["*_assembly_test.cc", "link_main_test.cc"])
]
cc_test(
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index ddcb1a1..f15ce20 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -54,12 +54,12 @@
# Demonstration executable
compile_benchmark_test(benchmark_test)
-add_test(NAME benchmark COMMAND benchmark_test --benchmark_min_time=0.01)
+add_test(benchmark benchmark_test --benchmark_min_time=0.01)
compile_benchmark_test(filter_test)
macro(add_filter_test name filter expect)
- add_test(NAME ${name} COMMAND filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect})
- add_test(NAME ${name}_list_only COMMAND filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect})
+ add_test(${name} filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect})
+ add_test(${name}_list_only filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect})
endmacro(add_filter_test)
add_filter_test(filter_simple "Foo" 3)
@@ -82,16 +82,16 @@
add_filter_test(filter_regex_end_negative "-.*Ba$" 4)
compile_benchmark_test(options_test)
-add_test(NAME options_benchmarks COMMAND options_test --benchmark_min_time=0.01)
+add_test(options_benchmarks options_test --benchmark_min_time=0.01)
compile_benchmark_test(basic_test)
-add_test(NAME basic_benchmark COMMAND basic_test --benchmark_min_time=0.01)
+add_test(basic_benchmark basic_test --benchmark_min_time=0.01)
compile_benchmark_test(diagnostics_test)
-add_test(NAME diagnostics_test COMMAND diagnostics_test --benchmark_min_time=0.01)
+add_test(diagnostics_test diagnostics_test --benchmark_min_time=0.01)
compile_benchmark_test(skip_with_error_test)
-add_test(NAME skip_with_error_test COMMAND skip_with_error_test --benchmark_min_time=0.01)
+add_test(skip_with_error_test skip_with_error_test --benchmark_min_time=0.01)
compile_benchmark_test(donotoptimize_test)
# Some of the issues with DoNotOptimize only occur when optimization is enabled
@@ -99,49 +99,46 @@
if (BENCHMARK_HAS_O3_FLAG)
set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3")
endif()
-add_test(NAME donotoptimize_test COMMAND donotoptimize_test --benchmark_min_time=0.01)
+add_test(donotoptimize_test donotoptimize_test --benchmark_min_time=0.01)
compile_benchmark_test(fixture_test)
-add_test(NAME fixture_test COMMAND fixture_test --benchmark_min_time=0.01)
+add_test(fixture_test fixture_test --benchmark_min_time=0.01)
compile_benchmark_test(register_benchmark_test)
-add_test(NAME register_benchmark_test COMMAND register_benchmark_test --benchmark_min_time=0.01)
+add_test(register_benchmark_test register_benchmark_test --benchmark_min_time=0.01)
compile_benchmark_test(map_test)
-add_test(NAME map_test COMMAND map_test --benchmark_min_time=0.01)
+add_test(map_test map_test --benchmark_min_time=0.01)
compile_benchmark_test(multiple_ranges_test)
-add_test(NAME multiple_ranges_test COMMAND multiple_ranges_test --benchmark_min_time=0.01)
+add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01)
compile_benchmark_test_with_main(link_main_test)
-add_test(NAME link_main_test COMMAND link_main_test --benchmark_min_time=0.01)
+add_test(link_main_test link_main_test --benchmark_min_time=0.01)
compile_output_test(reporter_output_test)
-add_test(NAME reporter_output_test COMMAND reporter_output_test --benchmark_min_time=0.01)
+add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01)
compile_output_test(templated_fixture_test)
-add_test(NAME templated_fixture_test COMMAND templated_fixture_test --benchmark_min_time=0.01)
+add_test(templated_fixture_test templated_fixture_test --benchmark_min_time=0.01)
compile_output_test(user_counters_test)
-add_test(NAME user_counters_test COMMAND user_counters_test --benchmark_min_time=0.01)
-
-compile_output_test(internal_threading_test)
-add_test(NAME internal_threading_test COMMAND internal_threading_test --benchmark_min_time=0.01)
+add_test(user_counters_test user_counters_test --benchmark_min_time=0.01)
compile_output_test(report_aggregates_only_test)
-add_test(NAME report_aggregates_only_test COMMAND report_aggregates_only_test --benchmark_min_time=0.01)
+add_test(report_aggregates_only_test report_aggregates_only_test --benchmark_min_time=0.01)
compile_output_test(display_aggregates_only_test)
-add_test(NAME display_aggregates_only_test COMMAND display_aggregates_only_test --benchmark_min_time=0.01)
+add_test(display_aggregates_only_test display_aggregates_only_test --benchmark_min_time=0.01)
compile_output_test(user_counters_tabular_test)
-add_test(NAME user_counters_tabular_test COMMAND user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01)
+add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01)
compile_output_test(user_counters_thousands_test)
-add_test(NAME user_counters_thousands_test COMMAND user_counters_thousands_test --benchmark_min_time=0.01)
+add_test(user_counters_thousands_test user_counters_thousands_test --benchmark_min_time=0.01)
compile_output_test(memory_manager_test)
-add_test(NAME memory_manager_test COMMAND memory_manager_test --benchmark_min_time=0.01)
+add_test(memory_manager_test memory_manager_test --benchmark_min_time=0.01)
check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG)
if (BENCHMARK_HAS_CXX03_FLAG)
@@ -159,7 +156,7 @@
PROPERTIES
LINK_FLAGS "-Wno-odr")
endif()
- add_test(NAME cxx03 COMMAND cxx03_test --benchmark_min_time=0.01)
+ add_test(cxx03 cxx03_test --benchmark_min_time=0.01)
endif()
# Attempt to work around flaky test failures when running on Appveyor servers.
@@ -169,7 +166,7 @@
set(COMPLEXITY_MIN_TIME "0.01")
endif()
compile_output_test(complexity_test)
-add_test(NAME complexity_benchmark COMMAND complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME})
+add_test(complexity_benchmark complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME})
###############################################################################
# GoogleTest Unit Tests
@@ -178,18 +175,22 @@
if (BENCHMARK_ENABLE_GTEST_TESTS)
macro(compile_gtest name)
add_executable(${name} "${name}.cc")
+ if (TARGET googletest)
+ add_dependencies(${name} googletest)
+ endif()
+ if (GTEST_INCLUDE_DIRS)
+ target_include_directories(${name} PRIVATE ${GTEST_INCLUDE_DIRS})
+ endif()
target_link_libraries(${name} benchmark
- gmock_main ${CMAKE_THREAD_LIBS_INIT})
+ ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
endmacro(compile_gtest)
macro(add_gtest name)
compile_gtest(${name})
- add_test(NAME ${name} COMMAND ${name})
+ add_test(${name} ${name})
endmacro()
add_gtest(benchmark_gtest)
- add_gtest(benchmark_name_gtest)
- add_gtest(commandlineflags_gtest)
add_gtest(statistics_gtest)
add_gtest(string_util_gtest)
endif(BENCHMARK_ENABLE_GTEST_TESTS)
diff --git a/test/basic_test.cc b/test/basic_test.cc
index 5f3dd1a..d07fbc0 100644
--- a/test/basic_test.cc
+++ b/test/basic_test.cc
@@ -98,7 +98,7 @@
void BM_KeepRunning(benchmark::State& state) {
- benchmark::IterationCount iter_count = 0;
+ size_t iter_count = 0;
assert(iter_count == state.iterations());
while (state.KeepRunning()) {
++iter_count;
@@ -109,8 +109,8 @@
void BM_KeepRunningBatch(benchmark::State& state) {
// Choose a prime batch size to avoid evenly dividing max_iterations.
- const benchmark::IterationCount batch_size = 101;
- benchmark::IterationCount iter_count = 0;
+ const size_t batch_size = 101;
+ size_t iter_count = 0;
while (state.KeepRunningBatch(batch_size)) {
iter_count += batch_size;
}
@@ -119,7 +119,7 @@
BENCHMARK(BM_KeepRunningBatch);
void BM_RangedFor(benchmark::State& state) {
- benchmark::IterationCount iter_count = 0;
+ size_t iter_count = 0;
for (auto _ : state) {
++iter_count;
}
diff --git a/test/benchmark_gtest.cc b/test/benchmark_gtest.cc
index 9557b20..10683b4 100644
--- a/test/benchmark_gtest.cc
+++ b/test/benchmark_gtest.cc
@@ -4,8 +4,6 @@
#include "gmock/gmock.h"
#include "gtest/gtest.h"
-namespace benchmark {
-namespace internal {
namespace {
TEST(AddRangeTest, Simple) {
@@ -32,97 +30,4 @@
EXPECT_THAT(dst, testing::ElementsAre(5, 8, 15));
}
-TEST(AddRangeTest, FullRange8) {
- std::vector<int8_t> dst;
- AddRange(&dst, int8_t{1}, std::numeric_limits<int8_t>::max(), 8);
- EXPECT_THAT(dst, testing::ElementsAre(1, 8, 64, 127));
-}
-
-TEST(AddRangeTest, FullRange64) {
- std::vector<int64_t> dst;
- AddRange(&dst, int64_t{1}, std::numeric_limits<int64_t>::max(), 1024);
- EXPECT_THAT(
- dst, testing::ElementsAre(1LL, 1024LL, 1048576LL, 1073741824LL,
- 1099511627776LL, 1125899906842624LL,
- 1152921504606846976LL, 9223372036854775807LL));
-}
-
-TEST(AddRangeTest, NegativeRanges) {
- std::vector<int> dst;
- AddRange(&dst, -8, 0, 2);
- EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1, 0));
-}
-
-TEST(AddRangeTest, StrictlyNegative) {
- std::vector<int> dst;
- AddRange(&dst, -8, -1, 2);
- EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1));
-}
-
-TEST(AddRangeTest, SymmetricNegativeRanges) {
- std::vector<int> dst;
- AddRange(&dst, -8, 8, 2);
- EXPECT_THAT(dst, testing::ElementsAre(-8, -4, -2, -1, 0, 1, 2, 4, 8));
-}
-
-TEST(AddRangeTest, SymmetricNegativeRangesOddMult) {
- std::vector<int> dst;
- AddRange(&dst, -30, 32, 5);
- EXPECT_THAT(dst, testing::ElementsAre(-30, -25, -5, -1, 0, 1, 5, 25, 32));
-}
-
-TEST(AddRangeTest, NegativeRangesAsymmetric) {
- std::vector<int> dst;
- AddRange(&dst, -3, 5, 2);
- EXPECT_THAT(dst, testing::ElementsAre(-3, -2, -1, 0, 1, 2, 4, 5));
-}
-
-TEST(AddRangeTest, NegativeRangesLargeStep) {
- // Always include -1, 0, 1 when crossing zero.
- std::vector<int> dst;
- AddRange(&dst, -8, 8, 10);
- EXPECT_THAT(dst, testing::ElementsAre(-8, -1, 0, 1, 8));
-}
-
-TEST(AddRangeTest, ZeroOnlyRange) {
- std::vector<int> dst;
- AddRange(&dst, 0, 0, 2);
- EXPECT_THAT(dst, testing::ElementsAre(0));
-}
-
-TEST(AddRangeTest, NegativeRange64) {
- std::vector<int64_t> dst;
- AddRange<int64_t>(&dst, -4, 4, 2);
- EXPECT_THAT(dst, testing::ElementsAre(-4, -2, -1, 0, 1, 2, 4));
-}
-
-TEST(AddRangeTest, NegativeRangePreservesExistingOrder) {
- // If elements already exist in the range, ensure we don't change
- // their ordering by adding negative values.
- std::vector<int64_t> dst = {1, 2, 3};
- AddRange<int64_t>(&dst, -2, 2, 2);
- EXPECT_THAT(dst, testing::ElementsAre(1, 2, 3, -2, -1, 0, 1, 2));
-}
-
-TEST(AddRangeTest, FullNegativeRange64) {
- std::vector<int64_t> dst;
- const auto min = std::numeric_limits<int64_t>::min();
- const auto max = std::numeric_limits<int64_t>::max();
- AddRange(&dst, min, max, 1024);
- EXPECT_THAT(
- dst, testing::ElementsAreArray(std::vector<int64_t>{
- min, -1152921504606846976LL, -1125899906842624LL,
- -1099511627776LL, -1073741824LL, -1048576LL, -1024LL, -1LL, 0LL,
- 1LL, 1024LL, 1048576LL, 1073741824LL, 1099511627776LL,
- 1125899906842624LL, 1152921504606846976LL, max}));
-}
-
-TEST(AddRangeTest, Simple8) {
- std::vector<int8_t> dst;
- AddRange<int8_t>(&dst, 1, 8, 2);
- EXPECT_THAT(dst, testing::ElementsAre(1, 2, 4, 8));
-}
-
-} // namespace
-} // namespace internal
-} // namespace benchmark
+} // end namespace
diff --git a/test/benchmark_name_gtest.cc b/test/benchmark_name_gtest.cc
deleted file mode 100644
index afb401c..0000000
--- a/test/benchmark_name_gtest.cc
+++ /dev/null
@@ -1,74 +0,0 @@
-#include "benchmark/benchmark.h"
-#include "gtest/gtest.h"
-
-namespace {
-
-using namespace benchmark;
-using namespace benchmark::internal;
-
-TEST(BenchmarkNameTest, Empty) {
- const auto name = BenchmarkName();
- EXPECT_EQ(name.str(), std::string());
-}
-
-TEST(BenchmarkNameTest, FunctionName) {
- auto name = BenchmarkName();
- name.function_name = "function_name";
- EXPECT_EQ(name.str(), "function_name");
-}
-
-TEST(BenchmarkNameTest, FunctionNameAndArgs) {
- auto name = BenchmarkName();
- name.function_name = "function_name";
- name.args = "some_args:3/4/5";
- EXPECT_EQ(name.str(), "function_name/some_args:3/4/5");
-}
-
-TEST(BenchmarkNameTest, MinTime) {
- auto name = BenchmarkName();
- name.function_name = "function_name";
- name.args = "some_args:3/4";
- name.min_time = "min_time:3.4s";
- EXPECT_EQ(name.str(), "function_name/some_args:3/4/min_time:3.4s");
-}
-
-TEST(BenchmarkNameTest, Iterations) {
- auto name = BenchmarkName();
- name.function_name = "function_name";
- name.min_time = "min_time:3.4s";
- name.iterations = "iterations:42";
- EXPECT_EQ(name.str(), "function_name/min_time:3.4s/iterations:42");
-}
-
-TEST(BenchmarkNameTest, Repetitions) {
- auto name = BenchmarkName();
- name.function_name = "function_name";
- name.min_time = "min_time:3.4s";
- name.repetitions = "repetitions:24";
- EXPECT_EQ(name.str(), "function_name/min_time:3.4s/repetitions:24");
-}
-
-TEST(BenchmarkNameTest, TimeType) {
- auto name = BenchmarkName();
- name.function_name = "function_name";
- name.min_time = "min_time:3.4s";
- name.time_type = "hammer_time";
- EXPECT_EQ(name.str(), "function_name/min_time:3.4s/hammer_time");
-}
-
-TEST(BenchmarkNameTest, Threads) {
- auto name = BenchmarkName();
- name.function_name = "function_name";
- name.min_time = "min_time:3.4s";
- name.threads = "threads:256";
- EXPECT_EQ(name.str(), "function_name/min_time:3.4s/threads:256");
-}
-
-TEST(BenchmarkNameTest, TestEmptyFunctionName) {
- auto name = BenchmarkName();
- name.args = "first:3/second:4";
- name.threads = "threads:22";
- EXPECT_EQ(name.str(), "first:3/second:4/threads:22");
-}
-
-} // end namespace
diff --git a/test/commandlineflags_gtest.cc b/test/commandlineflags_gtest.cc
deleted file mode 100644
index 36bdb44..0000000
--- a/test/commandlineflags_gtest.cc
+++ /dev/null
@@ -1,201 +0,0 @@
-#include <cstdlib>
-
-#include "../src/commandlineflags.h"
-#include "../src/internal_macros.h"
-#include "gtest/gtest.h"
-
-namespace benchmark {
-namespace {
-
-#if defined(BENCHMARK_OS_WINDOWS)
-int setenv(const char* name, const char* value, int overwrite) {
- if (!overwrite) {
- // NOTE: getenv_s is far superior but not available under mingw.
- char* env_value = getenv(name);
- if (env_value == nullptr) {
- return -1;
- }
- }
- return _putenv_s(name, value);
-}
-
-int unsetenv(const char* name) {
- return _putenv_s(name, "");
-}
-
-#endif // BENCHMARK_OS_WINDOWS
-
-TEST(BoolFromEnv, Default) {
- ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0);
- EXPECT_EQ(BoolFromEnv("not_in_env", true), true);
-}
-
-TEST(BoolFromEnv, False) {
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "N", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "n", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "NO", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "No", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "no", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "F", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "f", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "FALSE", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "False", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "false", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "OFF", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Off", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "off", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", true), false);
- unsetenv("BENCHMARK_IN_ENV");
-}
-
-TEST(BoolFromEnv, True) {
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "1", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Y", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "y", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "YES", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "Yes", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "yes", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "T", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "t", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "TRUE", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "True", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "true", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "ON", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "On", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "on", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-
-#ifndef BENCHMARK_OS_WINDOWS
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "", 1), 0);
- EXPECT_EQ(BoolFromEnv("in_env", false), true);
- unsetenv("BENCHMARK_IN_ENV");
-#endif
-}
-
-TEST(Int32FromEnv, NotInEnv) {
- ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0);
- EXPECT_EQ(Int32FromEnv("not_in_env", 42), 42);
-}
-
-TEST(Int32FromEnv, InvalidInteger) {
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0);
- EXPECT_EQ(Int32FromEnv("in_env", 42), 42);
- unsetenv("BENCHMARK_IN_ENV");
-}
-
-TEST(Int32FromEnv, ValidInteger) {
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "42", 1), 0);
- EXPECT_EQ(Int32FromEnv("in_env", 64), 42);
- unsetenv("BENCHMARK_IN_ENV");
-}
-
-TEST(DoubleFromEnv, NotInEnv) {
- ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0);
- EXPECT_EQ(DoubleFromEnv("not_in_env", 0.51), 0.51);
-}
-
-TEST(DoubleFromEnv, InvalidReal) {
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0);
- EXPECT_EQ(DoubleFromEnv("in_env", 0.51), 0.51);
- unsetenv("BENCHMARK_IN_ENV");
-}
-
-TEST(DoubleFromEnv, ValidReal) {
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "0.51", 1), 0);
- EXPECT_EQ(DoubleFromEnv("in_env", 0.71), 0.51);
- unsetenv("BENCHMARK_IN_ENV");
-}
-
-TEST(StringFromEnv, Default) {
- ASSERT_EQ(unsetenv("BENCHMARK_NOT_IN_ENV"), 0);
- EXPECT_STREQ(StringFromEnv("not_in_env", "foo"), "foo");
-}
-
-TEST(StringFromEnv, Valid) {
- ASSERT_EQ(setenv("BENCHMARK_IN_ENV", "foo", 1), 0);
- EXPECT_STREQ(StringFromEnv("in_env", "bar"), "foo");
- unsetenv("BENCHMARK_IN_ENV");
-}
-
-} // namespace
-} // namespace benchmark
diff --git a/test/complexity_test.cc b/test/complexity_test.cc
index 5681fdc..323ddfe 100644
--- a/test/complexity_test.cc
+++ b/test/complexity_test.cc
@@ -28,8 +28,6 @@
AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"},
{"\"run_name\": \"%name\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": %int,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"BigO\",$", MR_Next},
{"\"cpu_coefficient\": %float,$", MR_Next},
{"\"real_coefficient\": %float,$", MR_Next},
@@ -39,8 +37,6 @@
{"\"name\": \"%rms_name\",$"},
{"\"run_name\": \"%name\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": %int,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"RMS\",$", MR_Next},
{"\"rms\": %float$", MR_Next},
{"}", MR_Next}});
@@ -66,9 +62,9 @@
}
BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity();
-BENCHMARK(BM_Complexity_O1)
- ->Range(1, 1 << 18)
- ->Complexity([](benchmark::IterationCount) { return 1.0; });
+BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int64_t) {
+ return 1.0;
+});
const char *one_test_name = "BM_Complexity_O1";
const char *big_o_1_test_name = "BM_Complexity_O1_BigO";
@@ -121,9 +117,7 @@
BENCHMARK(BM_Complexity_O_N)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
- ->Complexity([](benchmark::IterationCount n) -> double {
- return static_cast<double>(n);
- });
+ ->Complexity([](int64_t n) -> double { return static_cast<double>(n); });
BENCHMARK(BM_Complexity_O_N)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
@@ -162,9 +156,7 @@
BENCHMARK(BM_Complexity_O_N_log_N)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
- ->Complexity([](benchmark::IterationCount n) {
- return kLog2E * n * log(static_cast<double>(n));
- });
+ ->Complexity([](int64_t n) { return kLog2E * n * log(static_cast<double>(n)); });
BENCHMARK(BM_Complexity_O_N_log_N)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
@@ -185,28 +177,6 @@
rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n);
// ========================================================================= //
-// -------- Testing formatting of Complexity with captured args ------------ //
-// ========================================================================= //
-
-void BM_ComplexityCaptureArgs(benchmark::State& state, int n) {
- for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
- }
- state.SetComplexityN(n);
-}
-
-BENCHMARK_CAPTURE(BM_ComplexityCaptureArgs, capture_test, 100)
- ->Complexity(benchmark::oN)
- ->Ranges({{1, 2}, {3, 4}});
-
-const std::string complexity_capture_name =
- "BM_ComplexityCaptureArgs/capture_test";
-
-ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO",
- complexity_capture_name + "_RMS", "N");
-
-// ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ //
// ========================================================================= //
diff --git a/test/cxx03_test.cc b/test/cxx03_test.cc
index c4c9a52..baa9ed9 100644
--- a/test/cxx03_test.cc
+++ b/test/cxx03_test.cc
@@ -14,7 +14,7 @@
void BM_empty(benchmark::State& state) {
while (state.KeepRunning()) {
- volatile benchmark::IterationCount x = state.iterations();
+ volatile std::size_t x = state.iterations();
((void)x);
}
}
diff --git a/test/internal_threading_test.cc b/test/internal_threading_test.cc
deleted file mode 100644
index 039d7c1..0000000
--- a/test/internal_threading_test.cc
+++ /dev/null
@@ -1,184 +0,0 @@
-
-#undef NDEBUG
-
-#include <chrono>
-#include <thread>
-#include "../src/timers.h"
-#include "benchmark/benchmark.h"
-#include "output_test.h"
-
-static const std::chrono::duration<double, std::milli> time_frame(50);
-static const double time_frame_in_sec(
- std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1, 1>>>(
- time_frame)
- .count());
-
-void MyBusySpinwait() {
- const auto start = benchmark::ChronoClockNow();
-
- while (true) {
- const auto now = benchmark::ChronoClockNow();
- const auto elapsed = now - start;
-
- if (std::chrono::duration<double, std::chrono::seconds::period>(elapsed) >=
- time_frame)
- return;
- }
-}
-
-// ========================================================================= //
-// --------------------------- TEST CASES BEGIN ---------------------------- //
-// ========================================================================= //
-
-// ========================================================================= //
-// BM_MainThread
-
-void BM_MainThread(benchmark::State& state) {
- for (auto _ : state) {
- MyBusySpinwait();
- state.SetIterationTime(time_frame_in_sec);
- }
- state.counters["invtime"] =
- benchmark::Counter{1, benchmark::Counter::kIsRate};
-}
-
-BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1);
-BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->UseRealTime();
-BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->UseManualTime();
-BENCHMARK(BM_MainThread)->Iterations(1)->Threads(1)->MeasureProcessCPUTime();
-BENCHMARK(BM_MainThread)
- ->Iterations(1)
- ->Threads(1)
- ->MeasureProcessCPUTime()
- ->UseRealTime();
-BENCHMARK(BM_MainThread)
- ->Iterations(1)
- ->Threads(1)
- ->MeasureProcessCPUTime()
- ->UseManualTime();
-
-BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2);
-BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->UseRealTime();
-BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->UseManualTime();
-BENCHMARK(BM_MainThread)->Iterations(1)->Threads(2)->MeasureProcessCPUTime();
-BENCHMARK(BM_MainThread)
- ->Iterations(1)
- ->Threads(2)
- ->MeasureProcessCPUTime()
- ->UseRealTime();
-BENCHMARK(BM_MainThread)
- ->Iterations(1)
- ->Threads(2)
- ->MeasureProcessCPUTime()
- ->UseManualTime();
-
-// ========================================================================= //
-// BM_WorkerThread
-
-void BM_WorkerThread(benchmark::State& state) {
- for (auto _ : state) {
- std::thread Worker(&MyBusySpinwait);
- Worker.join();
- state.SetIterationTime(time_frame_in_sec);
- }
- state.counters["invtime"] =
- benchmark::Counter{1, benchmark::Counter::kIsRate};
-}
-
-BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1);
-BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->UseRealTime();
-BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->UseManualTime();
-BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(1)->MeasureProcessCPUTime();
-BENCHMARK(BM_WorkerThread)
- ->Iterations(1)
- ->Threads(1)
- ->MeasureProcessCPUTime()
- ->UseRealTime();
-BENCHMARK(BM_WorkerThread)
- ->Iterations(1)
- ->Threads(1)
- ->MeasureProcessCPUTime()
- ->UseManualTime();
-
-BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2);
-BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->UseRealTime();
-BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->UseManualTime();
-BENCHMARK(BM_WorkerThread)->Iterations(1)->Threads(2)->MeasureProcessCPUTime();
-BENCHMARK(BM_WorkerThread)
- ->Iterations(1)
- ->Threads(2)
- ->MeasureProcessCPUTime()
- ->UseRealTime();
-BENCHMARK(BM_WorkerThread)
- ->Iterations(1)
- ->Threads(2)
- ->MeasureProcessCPUTime()
- ->UseManualTime();
-
-// ========================================================================= //
-// BM_MainThreadAndWorkerThread
-
-void BM_MainThreadAndWorkerThread(benchmark::State& state) {
- for (auto _ : state) {
- std::thread Worker(&MyBusySpinwait);
- MyBusySpinwait();
- Worker.join();
- state.SetIterationTime(time_frame_in_sec);
- }
- state.counters["invtime"] =
- benchmark::Counter{1, benchmark::Counter::kIsRate};
-}
-
-BENCHMARK(BM_MainThreadAndWorkerThread)->Iterations(1)->Threads(1);
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(1)
- ->UseRealTime();
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(1)
- ->UseManualTime();
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(1)
- ->MeasureProcessCPUTime();
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(1)
- ->MeasureProcessCPUTime()
- ->UseRealTime();
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(1)
- ->MeasureProcessCPUTime()
- ->UseManualTime();
-
-BENCHMARK(BM_MainThreadAndWorkerThread)->Iterations(1)->Threads(2);
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(2)
- ->UseRealTime();
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(2)
- ->UseManualTime();
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(2)
- ->MeasureProcessCPUTime();
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(2)
- ->MeasureProcessCPUTime()
- ->UseRealTime();
-BENCHMARK(BM_MainThreadAndWorkerThread)
- ->Iterations(1)
- ->Threads(2)
- ->MeasureProcessCPUTime()
- ->UseManualTime();
-
-// ========================================================================= //
-// ---------------------------- TEST CASES END ----------------------------- //
-// ========================================================================= //
-
-int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
diff --git a/test/memory_manager_test.cc b/test/memory_manager_test.cc
index 90bed16..94be608 100644
--- a/test/memory_manager_test.cc
+++ b/test/memory_manager_test.cc
@@ -23,9 +23,6 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"},
{"\"run_name\": \"BM_empty\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -35,7 +32,8 @@
{"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_empty\",%csv_report$"}});
-int main(int argc, char* argv[]) {
+
+int main(int argc, char *argv[]) {
std::unique_ptr<benchmark::MemoryManager> mm(new TestMemoryManager());
benchmark::RegisterMemoryManager(mm.get());
diff --git a/test/multiple_ranges_test.cc b/test/multiple_ranges_test.cc
index b25f40e..c64acab 100644
--- a/test/multiple_ranges_test.cc
+++ b/test/multiple_ranges_test.cc
@@ -40,7 +40,8 @@
// NOTE: This is not TearDown as we want to check after _all_ runs are
// complete.
virtual ~MultipleRangesFixture() {
- if (actualValues != expectedValues) {
+ assert(actualValues.size() == expectedValues.size());
+ if (actualValues.size() != expectedValues.size()) {
std::cout << "EXPECTED\n";
for (auto v : expectedValues) {
std::cout << "{";
diff --git a/test/options_test.cc b/test/options_test.cc
index 7bfc235..fdec691 100644
--- a/test/options_test.cc
+++ b/test/options_test.cc
@@ -35,16 +35,6 @@
BENCHMARK(BM_basic)->ThreadRange(2, 4);
BENCHMARK(BM_basic)->ThreadPerCpu();
BENCHMARK(BM_basic)->Repetitions(3);
-BENCHMARK(BM_basic)
- ->RangeMultiplier(std::numeric_limits<int>::max())
- ->Range(std::numeric_limits<int64_t>::min(),
- std::numeric_limits<int64_t>::max());
-
-// Negative ranges
-BENCHMARK(BM_basic)->Range(-64, -1);
-BENCHMARK(BM_basic)->RangeMultiplier(4)->Range(-8, 8);
-BENCHMARK(BM_basic)->DenseRange(-2, 2, 1);
-BENCHMARK(BM_basic)->Ranges({{-64, 1}, {-8, -1}});
void CustomArgs(benchmark::internal::Benchmark* b) {
for (int i = 0; i < 10; ++i) {
diff --git a/test/output_test_helper.cc b/test/output_test_helper.cc
index bdb34c8..5dc951d 100644
--- a/test/output_test_helper.cc
+++ b/test/output_test_helper.cc
@@ -373,12 +373,6 @@
return 0;
}
-// Disable deprecated warnings temporarily because we need to reference
-// CSVReporter but don't want to trigger -Werror=-Wdeprecated
-#ifdef __GNUC__
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated"
-#endif
void RunOutputTests(int argc, char* argv[]) {
using internal::GetTestCaseList;
benchmark::Initialize(&argc, argv);
@@ -437,10 +431,6 @@
internal::GetResultsChecker().CheckResults(csv.out_stream);
}
-#ifdef __GNUC__
-#pragma GCC diagnostic pop
-#endif
-
int SubstrCnt(const std::string& haystack, const std::string& pat) {
if (pat.length() == 0) return 0;
int count = 0;
diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc
index 8486d59..ec6d51b 100644
--- a/test/reporter_output_test.cc
+++ b/test/reporter_output_test.cc
@@ -74,9 +74,6 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
{"\"run_name\": \"BM_basic\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -90,8 +87,6 @@
void BM_bytes_per_second(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
state.SetBytesProcessed(1);
}
@@ -102,9 +97,6 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
{"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -119,8 +111,6 @@
void BM_items_per_second(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
state.SetItemsProcessed(1);
}
@@ -131,9 +121,6 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
{"\"run_name\": \"BM_items_per_second\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -157,9 +144,6 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
{"\"run_name\": \"BM_label\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -183,9 +167,6 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
{"\"run_name\": \"BM_error\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"error_occurred\": true,$", MR_Next},
{"\"error_message\": \"message\",$", MR_Next}});
@@ -204,10 +185,7 @@
ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
{"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
- {"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next}});
+ {"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
// ========================================================================= //
@@ -222,10 +200,7 @@
ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
{"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
- {"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next}});
+ {"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
// ========================================================================= //
@@ -242,10 +217,7 @@
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
{"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
- {"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next}});
+ {"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
// ========================================================================= //
@@ -266,8 +238,6 @@
void BM_Complexity_O1(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
state.SetComplexityN(state.range(0));
}
@@ -297,34 +267,22 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"repetition_index\": 1,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_median\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
{"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
@@ -344,40 +302,25 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"repetition_index\": 1,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"repetition_index\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_median\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
{"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
@@ -399,46 +342,28 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 4,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 4,$", MR_Next},
- {"\"repetition_index\": 1,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 4,$", MR_Next},
- {"\"repetition_index\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 4,$", MR_Next},
- {"\"repetition_index\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 4,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_median\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 4,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
{"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 4,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 4,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
@@ -459,10 +384,7 @@
ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
{"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
- {"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 1,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next}});
+ {"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
// Test that non-aggregate data is not reported
@@ -482,22 +404,16 @@
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
{"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
@@ -524,22 +440,16 @@
{"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
{"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut,
@@ -570,24 +480,18 @@
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
{"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"time_unit\": \"us\",?$"}});
@@ -639,35 +543,24 @@
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"repetition_index\": 1,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"repetition_index\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": 5,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
@@ -675,8 +568,6 @@
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
@@ -684,8 +575,6 @@
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
@@ -693,8 +582,6 @@
{"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 3,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"\",$", MR_Next},
{"\"iterations\": 3,$", MR_Next},
{"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
@@ -711,37 +598,6 @@
{"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
// ========================================================================= //
-// ------------------------- Testing StrEscape JSON ------------------------ //
-// ========================================================================= //
-#if 0 // enable when csv testing code correctly handles multi-line fields
-void BM_JSON_Format(benchmark::State& state) {
- state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
- for (auto _ : state) {
- }
-}
-BENCHMARK(BM_JSON_Format);
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
- {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
- {"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
- {"\"error_occurred\": true,$", MR_Next},
- {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
-#endif
-// ========================================================================= //
-// -------------------------- Testing CsvEscape ---------------------------- //
-// ========================================================================= //
-
-void BM_CSV_Format(benchmark::State& state) {
- state.SkipWithError("\"freedom\"");
- for (auto _ : state) {
- }
-}
-BENCHMARK(BM_CSV_Format);
-ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
-
-// ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ //
// ========================================================================= //
diff --git a/test/state_assembly_test.cc b/test/state_assembly_test.cc
index 7ddbb3b..abe9a4d 100644
--- a/test/state_assembly_test.cc
+++ b/test/state_assembly_test.cc
@@ -25,7 +25,7 @@
for (auto _ : S) {
// CHECK: .L[[LOOP_HEAD:[a-zA-Z0-9_]+]]:
// CHECK-GNU-NEXT: subq $1, %rbx
- // CHECK-CLANG-NEXT: {{(addq \$1, %rax|incq %rax|addq \$-1, %rbx)}}
+ // CHECK-CLANG-NEXT: {{(addq \$1,|incq)}} %rax
// CHECK-NEXT: jne .L[[LOOP_HEAD]]
benchmark::DoNotOptimize(x);
}
diff --git a/test/string_util_gtest.cc b/test/string_util_gtest.cc
index 01bf155..2c5d073 100644
--- a/test/string_util_gtest.cc
+++ b/test/string_util_gtest.cc
@@ -3,7 +3,6 @@
//===---------------------------------------------------------------------===//
#include "../src/string_util.h"
-#include "../src/internal_macros.h"
#include "gtest/gtest.h"
namespace {
@@ -61,11 +60,9 @@
EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16));
EXPECT_EQ(4ul, pos);
}
-#ifndef BENCHMARK_HAS_NO_EXCEPTIONS
{
ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument);
}
-#endif
}
TEST(StringUtilTest, stoi) {
@@ -109,11 +106,9 @@
EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16));
EXPECT_EQ(4ul, pos);
}
-#ifndef BENCHMARK_HAS_NO_EXCEPTIONS
{
ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument);
}
-#endif
}
TEST(StringUtilTest, stod) {
@@ -143,11 +138,9 @@
EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos));
EXPECT_EQ(8ul, pos);
}
-#ifndef BENCHMARK_HAS_NO_EXCEPTIONS
{
ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument);
}
-#endif
}
} // end namespace
diff --git a/test/user_counters_tabular_test.cc b/test/user_counters_tabular_test.cc
index 18373c0..030e989 100644
--- a/test/user_counters_tabular_test.cc
+++ b/test/user_counters_tabular_test.cc
@@ -73,9 +73,6 @@
{{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
{"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -107,8 +104,6 @@
void BM_CounterRates_Tabular(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
namespace bm = benchmark;
state.counters.insert({
@@ -126,9 +121,6 @@
{"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -176,9 +168,6 @@
{{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
{"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -214,9 +203,6 @@
{{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
{"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -256,9 +242,6 @@
{{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
{"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
diff --git a/test/user_counters_test.cc b/test/user_counters_test.cc
index 5699f4f..bb0d6b4 100644
--- a/test/user_counters_test.cc
+++ b/test/user_counters_test.cc
@@ -34,9 +34,6 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
{"\"run_name\": \"BM_Counters_Simple\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -64,8 +61,6 @@
}
void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
state.counters["foo"] = 1;
state.counters["bar"] = ++num_calls1;
@@ -80,9 +75,6 @@
{{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
{"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -113,8 +105,6 @@
void BM_Counters_Rate(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
namespace bm = benchmark;
state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate};
@@ -127,9 +117,6 @@
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
{"\"run_name\": \"BM_Counters_Rate\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -149,89 +136,6 @@
CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate);
// ========================================================================= //
-// ----------------------- Inverted Counters Output ------------------------ //
-// ========================================================================= //
-
-void BM_Invert(benchmark::State& state) {
- for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
- }
- namespace bm = benchmark;
- state.counters["foo"] = bm::Counter{0.0001, bm::Counter::kInvert};
- state.counters["bar"] = bm::Counter{10000, bm::Counter::kInvert};
-}
-BENCHMARK(BM_Invert);
-ADD_CASES(TC_ConsoleOut,
- {{"^BM_Invert %console_report bar=%hrfloatu foo=%hrfloatk$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Invert\",$"},
- {"\"run_name\": \"BM_Invert\",$", MR_Next},
- {"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bar\": %float,$", MR_Next},
- {"\"foo\": %float$", MR_Next},
- {"}", MR_Next}});
-ADD_CASES(TC_CSVOut, {{"^\"BM_Invert\",%csv_report,%float,%float$"}});
-// VS2013 does not allow this function to be passed as a lambda argument
-// to CHECK_BENCHMARK_RESULTS()
-void CheckInvert(Results const& e) {
- CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 10000, 0.0001);
- CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 0.0001, 0.0001);
-}
-CHECK_BENCHMARK_RESULTS("BM_Invert", &CheckInvert);
-
-// ========================================================================= //
-// ------------------------- InvertedRate Counters Output
-// -------------------------- //
-// ========================================================================= //
-
-void BM_Counters_InvertedRate(benchmark::State& state) {
- for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
- }
- namespace bm = benchmark;
- state.counters["foo"] =
- bm::Counter{1, bm::Counter::kIsRate | bm::Counter::kInvert};
- state.counters["bar"] =
- bm::Counter{8192, bm::Counter::kIsRate | bm::Counter::kInvert};
-}
-BENCHMARK(BM_Counters_InvertedRate);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_InvertedRate %console_report "
- "bar=%hrfloats foo=%hrfloats$"}});
-ADD_CASES(TC_JSONOut,
- {{"\"name\": \"BM_Counters_InvertedRate\",$"},
- {"\"run_name\": \"BM_Counters_InvertedRate\",$", MR_Next},
- {"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bar\": %float,$", MR_Next},
- {"\"foo\": %float$", MR_Next},
- {"}", MR_Next}});
-ADD_CASES(TC_CSVOut,
- {{"^\"BM_Counters_InvertedRate\",%csv_report,%float,%float$"}});
-// VS2013 does not allow this function to be passed as a lambda argument
-// to CHECK_BENCHMARK_RESULTS()
-void CheckInvertedRate(Results const& e) {
- double t = e.DurationCPUTime(); // this (and not real time) is the time used
- // check that the values are within 0.1% of the expected values
- CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, t, 0.001);
- CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, t / 8192.0, 0.001);
-}
-CHECK_BENCHMARK_RESULTS("BM_Counters_InvertedRate", &CheckInvertedRate);
-
-// ========================================================================= //
// ------------------------- Thread Counters Output ------------------------ //
// ========================================================================= //
@@ -248,9 +152,6 @@
{{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
{"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -287,9 +188,6 @@
{{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
{"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -315,8 +213,6 @@
void BM_Counters_AvgThreadsRate(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
namespace bm = benchmark;
state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate};
@@ -330,9 +226,6 @@
{"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -369,9 +262,6 @@
{{"\"name\": \"BM_Counters_IterationInvariant\",$"},
{"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -398,8 +288,6 @@
void BM_Counters_kIsIterationInvariantRate(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
namespace bm = benchmark;
state.counters["foo"] =
@@ -415,9 +303,6 @@
{"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$",
MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -457,9 +342,6 @@
{{"\"name\": \"BM_Counters_AvgIterations\",$"},
{"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -485,8 +367,6 @@
void BM_Counters_kAvgIterationsRate(benchmark::State& state) {
for (auto _ : state) {
- // This test requires a non-zero CPU time to avoid divide-by-zero
- benchmark::DoNotOptimize(state.iterations());
}
namespace bm = benchmark;
state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgIterationsRate};
@@ -500,9 +380,6 @@
{{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"},
{"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 0,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
diff --git a/test/user_counters_thousands_test.cc b/test/user_counters_thousands_test.cc
index 21d8285..fa0ef97 100644
--- a/test/user_counters_thousands_test.cc
+++ b/test/user_counters_thousands_test.cc
@@ -53,9 +53,6 @@
{{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"},
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"repetition_index\": 0,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -70,9 +67,6 @@
{{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"},
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"iteration\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"repetition_index\": 1,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -87,8 +81,6 @@
{{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"},
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"mean\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
@@ -104,8 +96,6 @@
{{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"},
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"median\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
@@ -121,8 +111,6 @@
{{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"},
{"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
{"\"run_type\": \"aggregate\",$", MR_Next},
- {"\"repetitions\": 2,$", MR_Next},
- {"\"threads\": 1,$", MR_Next},
{"\"aggregate_name\": \"stddev\",$", MR_Next},
{"\"iterations\": 2,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
diff --git a/tools/gbench/Inputs/test1_run1.json b/tools/gbench/Inputs/test1_run1.json
index 601e327..d7ec6a9 100644
--- a/tools/gbench/Inputs/test1_run1.json
+++ b/tools/gbench/Inputs/test1_run1.json
@@ -85,24 +85,7 @@
"time_unit": "ns"
},
{
- "name": "MyComplexityTest_BigO",
- "run_name": "MyComplexityTest",
- "run_type": "aggregate",
- "aggregate_name": "BigO",
- "cpu_coefficient": 4.2749856294592886e+00,
- "real_coefficient": 6.4789275289789780e+00,
- "big_o": "N",
- "time_unit": "ns"
- },
- {
- "name": "MyComplexityTest_RMS",
- "run_name": "MyComplexityTest",
- "run_type": "aggregate",
- "aggregate_name": "RMS",
- "rms": 4.5097802512472874e-03
- },
- {
- "name": "BM_NotBadTimeUnit",
+ "name": "BM_BadTimeUnit",
"iterations": 1000,
"real_time": 0.4,
"cpu_time": 0.5,
diff --git a/tools/gbench/Inputs/test1_run2.json b/tools/gbench/Inputs/test1_run2.json
index 3cbcf39..59a5ffa 100644
--- a/tools/gbench/Inputs/test1_run2.json
+++ b/tools/gbench/Inputs/test1_run2.json
@@ -85,24 +85,7 @@
"time_unit": "ns"
},
{
- "name": "MyComplexityTest_BigO",
- "run_name": "MyComplexityTest",
- "run_type": "aggregate",
- "aggregate_name": "BigO",
- "cpu_coefficient": 5.6215779594361486e+00,
- "real_coefficient": 5.6288314793554610e+00,
- "big_o": "N",
- "time_unit": "ns"
- },
- {
- "name": "MyComplexityTest_RMS",
- "run_name": "MyComplexityTest",
- "run_type": "aggregate",
- "aggregate_name": "RMS",
- "rms": 3.3128901852342174e-03
- },
- {
- "name": "BM_NotBadTimeUnit",
+ "name": "BM_BadTimeUnit",
"iterations": 1000,
"real_time": 0.04,
"cpu_time": 0.6,
diff --git a/tools/gbench/report.py b/tools/gbench/report.py
index 5bd3a8d..5085b93 100644
--- a/tools/gbench/report.py
+++ b/tools/gbench/report.py
@@ -114,10 +114,6 @@
return [x for x in list1 if x in list2]
-def is_potentially_comparable_benchmark(x):
- return ('time_unit' in x and 'real_time' in x and 'cpu_time' in x)
-
-
def partition_benchmarks(json1, json2):
"""
While preserving the ordering, find benchmarks with the same names in
@@ -129,17 +125,10 @@
names = intersect(json1_unique_names, json2_unique_names)
partitions = []
for name in names:
- time_unit = None
# Pick the time unit from the first entry of the lhs benchmark.
- # We should be careful not to crash with unexpected input.
- for x in json1['benchmarks']:
- if (x['name'] == name and is_potentially_comparable_benchmark(x)):
- time_unit = x['time_unit']
- break
- if time_unit is None:
- continue
+ time_unit = (x['time_unit']
+ for x in json1['benchmarks'] if x['name'] == name).next()
# Filter by name and time unit.
- # All the repetitions are assumed to be comparable.
lhs = [x for x in json1['benchmarks'] if x['name'] == name and
x['time_unit'] == time_unit]
rhs = [x for x in json2['benchmarks'] if x['name'] == name and
@@ -154,7 +143,11 @@
rhs = [x[field_name] for x in partition[1]]
return [lhs, rhs]
-def calc_utest(timings_cpu, timings_time):
+
+def print_utest(partition, utest_alpha, first_col_width, use_color=True):
+ timings_time = extract_field(partition, 'real_time')
+ timings_cpu = extract_field(partition, 'cpu_time')
+
min_rep_cnt = min(len(timings_time[0]),
len(timings_time[1]),
len(timings_cpu[0]),
@@ -162,33 +155,21 @@
# Does *everything* has at least UTEST_MIN_REPETITIONS repetitions?
if min_rep_cnt < UTEST_MIN_REPETITIONS:
- return False, None, None
+ return []
+
+ def get_utest_color(pval):
+ return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
time_pvalue = mannwhitneyu(
timings_time[0], timings_time[1], alternative='two-sided').pvalue
cpu_pvalue = mannwhitneyu(
timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue
- return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue
-
-def print_utest(partition, utest_alpha, first_col_width, use_color=True):
- def get_utest_color(pval):
- return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
-
- timings_time = extract_field(partition, 'real_time')
- timings_cpu = extract_field(partition, 'cpu_time')
- have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(timings_cpu, timings_time)
-
- # Check if we failed miserably with minimum required repetitions for utest
- if not have_optimal_repetitions and cpu_pvalue is None and time_pvalue is None:
- return []
-
dsc = "U Test, Repetitions: {} vs {}".format(
len(timings_cpu[0]), len(timings_cpu[1]))
dsc_color = BC_OKGREEN
- # We still got some results to show but issue a warning about it.
- if not have_optimal_repetitions:
+ if min_rep_cnt < UTEST_OPTIMAL_REPETITIONS:
dsc_color = BC_WARNING
dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format(
UTEST_OPTIMAL_REPETITIONS)
@@ -349,7 +330,7 @@
['BM_10PercentCPUToTime', '+0.1000',
'-0.1000', '100', '110', '100', '90'],
['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'],
- ['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'],
+ ['BM_BadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'],
]
json1, json2 = self.load_results()
output_lines_with_header = generate_difference_report(
