Upgrade libfuzzer-sys to 0.4.5
This project was upgraded with external_updater.
Usage: tools/external_updater/updater.sh update rust/crates/libfuzzer-sys
For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md
Test: TreeHugger
Change-Id: Ief01d01cb4ca209ce0244f96641034f481e589b2
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json
index 4d6cfd1..2b01961 100644
--- a/.cargo_vcs_info.json
+++ b/.cargo_vcs_info.json
@@ -1,5 +1,6 @@
{
"git": {
- "sha1": "a89115ac1105fa0c7c7d9cb5cdb479af36031aff"
- }
-}
+ "sha1": "396dc4ca1c8b4909ad24985dbe83a326ddfd2b82"
+ },
+ "path_in_vcs": ""
+}
\ No newline at end of file
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 47e2bf2..5888af1 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -20,5 +20,7 @@
rustup component add rustfmt --toolchain stable
cargo +stable fmt --all -- --check
+ - run: cargo install cargo-fuzz
+
- name: Run tests
run: ./ci/script.sh
diff --git a/.gitignore b/.gitignore
index a9d37c5..4141263 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
target
Cargo.lock
+corpus
diff --git a/Android.bp b/Android.bp
index fbfcb90..b24a7aa 100644
--- a/Android.bp
+++ b/Android.bp
@@ -39,7 +39,7 @@
host_supported: true,
crate_name: "libfuzzer_sys",
cargo_env_compat: true,
- cargo_pkg_version: "0.4.2",
+ cargo_pkg_version: "0.4.5",
srcs: ["src/lib.rs"],
edition: "2018",
features: ["arbitrary-derive"],
@@ -47,4 +47,8 @@
"libarbitrary",
"libonce_cell",
],
+ apex_available: [
+ "//apex_available:platform",
+ "//apex_available:anyapex",
+ ],
}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 77be1fd..a924906 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,58 @@
--------------------------------------------------------------------------------
+## 0.4.5
+
+Released 2022-10-18.
+
+### Added
+
+* Added the ability to tell libfuzzer whether to keep files inputs in the corpus
+ or not. See the `Corpus` type and extended documentation for the
+ `fuzz_target!` macro for details.
+
+### Changed
+
+* Ensured that there is always at least one inline-never frame on the stack per
+ fuzz target. This helps prevent oss-fuzz from "deduplicating" different bugs
+ from different fuzz targets into the same bug.
+
+--------------------------------------------------------------------------------
+
+## 0.4.4
+
+Released 2022-09-01.
+
+### Changed
+
+* Updated to `libFuzzer` commit `df90d22` (`release/15.x`).
+* LLVM 16's [upcoming change][llvm_cxx17] to build requirements to C++17
+ necessitate reflecting those changes. (`libFuzzer` updates contain C++14 code)
+* Drastically reduce build times by using parallel C++ compilation jobs
+* Updated `rand` dependency from 0.8.3 to 0.8.5
+* Updated `flate2` dependency from 1.0.20 to 1.0.24
+
+[llvm_cxx17]: https://llvm.org/docs/ReleaseNotes.html#update-on-required-toolchains-to-build-llvm
+
+--------------------------------------------------------------------------------
+
+## 0.4.3
+
+Released 2020-03-03.
+
+### Changed
+
+* Updated to `libFuzzer` commit `60e32a1`.
+
+### Fixed
+
+* Fixed an issue where the `fuzz_target!` macro would sometimes expand to
+ versions of itself that were not `$crate` prefixed and would result in "error:
+ cannot find macro `fuzz_target` in this scope" if the caller didn't import the
+ macro but used the qualified version of it instead.
+
+--------------------------------------------------------------------------------
+
## 0.4.2
Released 2020-05-26.
diff --git a/Cargo.toml b/Cargo.toml
index 4be2136..cee7559 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,34 +3,38 @@
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
-# to registry (e.g., crates.io) dependencies
+# to registry (e.g., crates.io) dependencies.
#
-# If you believe there's an error in this file please file an
-# issue against the rust-lang/cargo repository. If you're
-# editing this file be aware that the upstream Cargo.toml
-# will likely look very different (and much more reasonable)
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
[package]
edition = "2018"
name = "libfuzzer-sys"
-version = "0.4.2"
+version = "0.4.5"
authors = ["The rust-fuzz Project Developers"]
description = "A wrapper around LLVM's libFuzzer runtime."
readme = "./README.md"
license = "MIT/Apache-2.0/NCSA"
repository = "https://github.com/rust-fuzz/libfuzzer"
+resolver = "1"
+
[dependencies.arbitrary]
version = "1"
[dependencies.once_cell]
version = "1"
+
[dev-dependencies.flate2]
-version = "1.0.20"
+version = "1.0.24"
[dev-dependencies.rand]
-version = "0.8.3"
+version = "0.8.5"
+
[build-dependencies.cc]
version = "1.0"
+features = ["parallel"]
[features]
arbitrary-derive = ["arbitrary/derive"]
diff --git a/Cargo.toml.orig b/Cargo.toml.orig
index 72285b9..bc5982d 100644
--- a/Cargo.toml.orig
+++ b/Cargo.toml.orig
@@ -6,25 +6,25 @@
name = "libfuzzer-sys"
readme = "./README.md"
repository = "https://github.com/rust-fuzz/libfuzzer"
-version = "0.4.2"
+version = "0.4.5"
[dependencies]
arbitrary = "1"
once_cell = "1"
[build-dependencies]
-cc = "1.0"
+cc = { version = "1.0", features = ["parallel"] }
[features]
arbitrary-derive = ["arbitrary/derive"]
[workspace]
members = [
- "./example",
- "./example_arbitrary",
- "./example_mutator",
+ "./example/fuzz",
+ "./example_arbitrary/fuzz",
+ "./example_mutator/fuzz",
]
[dev-dependencies]
-flate2 = "1.0.20"
-rand = "0.8.3"
+flate2 = "1.0.24"
+rand = "0.8.5"
diff --git a/METADATA b/METADATA
index abca0eb..0e7f01d 100644
--- a/METADATA
+++ b/METADATA
@@ -1,3 +1,7 @@
+# This project was upgraded with external_updater.
+# Usage: tools/external_updater/updater.sh update rust/crates/libfuzzer-sys
+# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md
+
name: "libfuzzer-sys"
description: "A wrapper around LLVM\'s libFuzzer runtime."
third_party {
@@ -7,13 +11,13 @@
}
url {
type: ARCHIVE
- value: "https://static.crates.io/crates/libfuzzer-sys/libfuzzer-sys-0.4.2.crate"
+ value: "https://static.crates.io/crates/libfuzzer-sys/libfuzzer-sys-0.4.5.crate"
}
- version: "0.4.2"
+ version: "0.4.5"
license_type: NOTICE
last_upgrade_date {
- year: 2021
- month: 6
- day: 21
+ year: 2022
+ month: 12
+ day: 12
}
}
diff --git a/build.rs b/build.rs
index 21c57c1..bf6c12f 100644
--- a/build.rs
+++ b/build.rs
@@ -26,7 +26,7 @@
println!("cargo:rerun-if-changed={}", source.display());
build.file(source.to_str().unwrap());
}
- build.flag("-std=c++11");
+ build.flag("-std=c++17");
build.flag("-fno-omit-frame-pointer");
build.flag("-w");
build.cpp(true);
diff --git a/ci/script.sh b/ci/script.sh
index 91fad80..59462d4 100755
--- a/ci/script.sh
+++ b/ci/script.sh
@@ -8,53 +8,26 @@
cargo test --doc
pushd ./example
-cargo rustc \
- --release \
- -- \
- -Cpasses='sancov' \
- -Cllvm-args=-sanitizer-coverage-level=3 \
- -Cllvm-args=-sanitizer-coverage-trace-compares \
- -Cllvm-args=-sanitizer-coverage-inline-8bit-counters \
- -Cllvm-args=-sanitizer-coverage-stack-depth \
- -Cllvm-args=-sanitizer-coverage-trace-geps \
- -Cllvm-args=-sanitizer-coverage-prune-blocks=0 \
- -Zsanitizer=address
-(! $CARGO_TARGET_DIR/release/example -runs=100000)
+cargo fuzz build
+cargo fuzz build --dev
+(! cargo fuzz run bananas -- -runs=100000)
popd
pushd ./example_arbitrary
-cargo rustc \
- --release \
- -- \
- -Cpasses='sancov' \
- -Cllvm-args=-sanitizer-coverage-level=3 \
- -Cllvm-args=-sanitizer-coverage-trace-compares \
- -Cllvm-args=-sanitizer-coverage-inline-8bit-counters \
- -Cllvm-args=-sanitizer-coverage-stack-depth \
- -Cllvm-args=-sanitizer-coverage-trace-geps \
- -Cllvm-args=-sanitizer-coverage-prune-blocks=0 \
- -Zsanitizer=address
-(! $CARGO_TARGET_DIR/release/example_arbitrary -runs=10000000)
+cargo fuzz build
+cargo fuzz build --dev
+(! cargo fuzz run rgb -- -runs=10000000)
RUST_LIBFUZZER_DEBUG_PATH=$(pwd)/debug_output \
- $CARGO_TARGET_DIR/release/example_arbitrary \
- $(ls ./crash-* | head -n 1)
+ cargo fuzz run rgb \
+ $(ls ./fuzz/artifacts/rgb/crash-* | head -n 1)
cat $(pwd)/debug_output
grep -q Rgb $(pwd)/debug_output
popd
pushd ./example_mutator
-cargo rustc \
- --release \
- -- \
- -Cpasses='sancov' \
- -Cllvm-args=-sanitizer-coverage-level=3 \
- -Cllvm-args=-sanitizer-coverage-trace-compares \
- -Cllvm-args=-sanitizer-coverage-inline-8bit-counters \
- -Cllvm-args=-sanitizer-coverage-stack-depth \
- -Cllvm-args=-sanitizer-coverage-trace-geps \
- -Cllvm-args=-sanitizer-coverage-prune-blocks=0 \
- -Zsanitizer=address
-(! $CARGO_TARGET_DIR/release/example_mutator -runs=10000000)
+cargo fuzz build
+cargo fuzz build --dev
+(! cargo fuzz run boom -- -runs=10000000)
popd
echo "All good!"
diff --git a/libfuzzer/CMakeLists.txt b/libfuzzer/CMakeLists.txt
index 3201ed2..a9a10f7 100644
--- a/libfuzzer/CMakeLists.txt
+++ b/libfuzzer/CMakeLists.txt
@@ -6,6 +6,8 @@
FuzzerExtFunctionsWeak.cpp
FuzzerExtFunctionsWindows.cpp
FuzzerExtraCounters.cpp
+ FuzzerExtraCountersDarwin.cpp
+ FuzzerExtraCountersWindows.cpp
FuzzerFork.cpp
FuzzerIO.cpp
FuzzerIOPosix.cpp
@@ -64,18 +66,19 @@
append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ LIBFUZZER_CFLAGS)
elseif(TARGET cxx-headers OR HAVE_LIBCXX)
# libFuzzer uses C++ standard library headers.
+ list(APPEND LIBFUZZER_CFLAGS ${COMPILER_RT_CXX_CFLAGS})
set(LIBFUZZER_DEPS cxx-headers)
endif()
append_list_if(COMPILER_RT_HAS_OMIT_FRAME_POINTER_FLAG -fno-omit-frame-pointer LIBFUZZER_CFLAGS)
if (CMAKE_CXX_FLAGS MATCHES "fsanitize-coverage")
- list(APPEND LIBFUZZER_CFLAGS -fno-sanitize-coverage=trace-pc-guard,edge,trace-cmp,indirect-calls,8bit-counters)
+ list(APPEND LIBFUZZER_CFLAGS -fsanitize-coverage=0)
endif()
if(MSVC)
# Silence warnings by turning off exceptions in MSVC headers and avoid an
- # error by unecessarily defining thread_local when it isn't even used on
+ # error by unnecessarily defining thread_local when it isn't even used on
# Windows.
list(APPEND LIBFUZZER_CFLAGS -D_HAS_EXCEPTIONS=0)
else()
@@ -136,15 +139,15 @@
COMPILER_RT_LIBCXX_PATH AND
COMPILER_RT_LIBCXXABI_PATH)
macro(partially_link_libcxx name dir arch)
- if(${arch} MATCHES "i386")
- set(EMULATION_ARGUMENT "-m" "elf_i386")
- else()
- set(EMULATION_ARGUMENT "")
+ get_target_flags_for_arch(${arch} target_cflags)
+ if(CMAKE_CXX_COMPILER_ID MATCHES Clang)
+ get_compiler_rt_target(${arch} target)
+ set(target_cflags --target=${target} ${target_cflags})
endif()
set(cxx_${arch}_merge_dir "${CMAKE_CURRENT_BINARY_DIR}/cxx_${arch}_merge.dir")
file(MAKE_DIRECTORY ${cxx_${arch}_merge_dir})
add_custom_command(TARGET clang_rt.${name}-${arch} POST_BUILD
- COMMAND ${CMAKE_LINKER} ${EMULATION_ARGUMENT} --whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" --no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
+ COMMAND ${CMAKE_CXX_COMPILER} ${target_cflags} -Wl,--whole-archive "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" -Wl,--no-whole-archive ${dir}/lib/libc++.a -r -o ${name}.o
COMMAND ${CMAKE_OBJCOPY} --localize-hidden ${name}.o
COMMAND ${CMAKE_COMMAND} -E remove "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>"
COMMAND ${CMAKE_AR} qcs "$<TARGET_LINKER_FILE:clang_rt.${name}-${arch}>" ${name}.o
@@ -160,7 +163,8 @@
CMAKE_ARGS -DCMAKE_CXX_COMPILER_WORKS=ON
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DLIBCXXABI_ENABLE_EXCEPTIONS=OFF
- -DLIBCXX_ABI_NAMESPACE=__Fuzzer)
+ -DLIBCXX_ABI_NAMESPACE=__Fuzzer
+ -DLIBCXX_ENABLE_EXCEPTIONS=OFF)
target_compile_options(RTfuzzer.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
add_dependencies(RTfuzzer.${arch} libcxx_fuzzer_${arch}-build)
target_compile_options(RTfuzzer_main.${arch} PRIVATE -isystem ${LIBCXX_${arch}_PREFIX}/include/c++/v1)
diff --git a/libfuzzer/FuzzerBuiltinsMsvc.h b/libfuzzer/FuzzerBuiltinsMsvc.h
index ab191b6..421dee7 100644
--- a/libfuzzer/FuzzerBuiltinsMsvc.h
+++ b/libfuzzer/FuzzerBuiltinsMsvc.h
@@ -41,7 +41,8 @@
#if !defined(_M_ARM) && !defined(_M_X64)
// Scan the high 32 bits.
if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X >> 32)))
- return static_cast<int>(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB.
+ return static_cast<int>(
+ 63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB.
// Scan the low 32 bits.
if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X)))
return static_cast<int>(63 - LeadZeroIdx);
diff --git a/libfuzzer/FuzzerCommand.h b/libfuzzer/FuzzerCommand.h
index 8730886..f653fe3 100644
--- a/libfuzzer/FuzzerCommand.h
+++ b/libfuzzer/FuzzerCommand.h
@@ -33,7 +33,7 @@
Command() : CombinedOutAndErr(false) {}
- explicit Command(const Vector<std::string> &ArgsToAdd)
+ explicit Command(const std::vector<std::string> &ArgsToAdd)
: Args(ArgsToAdd), CombinedOutAndErr(false) {}
explicit Command(const Command &Other)
@@ -58,7 +58,7 @@
// Gets all of the current command line arguments, **including** those after
// "-ignore-remaining-args=1".
- const Vector<std::string> &getArguments() const { return Args; }
+ const std::vector<std::string> &getArguments() const { return Args; }
// Adds the given argument before "-ignore_remaining_args=1", or at the end
// if that flag isn't present.
@@ -68,7 +68,7 @@
// Adds all given arguments before "-ignore_remaining_args=1", or at the end
// if that flag isn't present.
- void addArguments(const Vector<std::string> &ArgsToAdd) {
+ void addArguments(const std::vector<std::string> &ArgsToAdd) {
Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end());
}
@@ -155,16 +155,16 @@
Command(Command &&Other) = delete;
Command &operator=(Command &&Other) = delete;
- Vector<std::string>::iterator endMutableArgs() {
+ std::vector<std::string>::iterator endMutableArgs() {
return std::find(Args.begin(), Args.end(), ignoreRemainingArgs());
}
- Vector<std::string>::const_iterator endMutableArgs() const {
+ std::vector<std::string>::const_iterator endMutableArgs() const {
return std::find(Args.begin(), Args.end(), ignoreRemainingArgs());
}
// The command arguments. Args[0] is the command name.
- Vector<std::string> Args;
+ std::vector<std::string> Args;
// True indicates stderr is redirected to stdout.
bool CombinedOutAndErr;
diff --git a/libfuzzer/FuzzerCorpus.h b/libfuzzer/FuzzerCorpus.h
index f8c1260..e01891e 100644
--- a/libfuzzer/FuzzerCorpus.h
+++ b/libfuzzer/FuzzerCorpus.h
@@ -39,13 +39,13 @@
bool MayDeleteFile = false;
bool Reduced = false;
bool HasFocusFunction = false;
- Vector<uint32_t> UniqFeatureSet;
- Vector<uint8_t> DataFlowTraceForFocusFunction;
+ std::vector<uint32_t> UniqFeatureSet;
+ std::vector<uint8_t> DataFlowTraceForFocusFunction;
// Power schedule.
bool NeedsEnergyUpdate = false;
double Energy = 0.0;
double SumIncidence = 0.0;
- Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
+ std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
// Delete feature Idx and its frequency from FeatureFreqs.
bool DeleteFeatureFreq(uint32_t Idx) {
@@ -209,7 +209,7 @@
InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
bool HasFocusFunction, bool NeverReduce,
std::chrono::microseconds TimeOfUnit,
- const Vector<uint32_t> &FeatureSet,
+ const std::vector<uint32_t> &FeatureSet,
const DataFlowTrace &DFT, const InputInfo *BaseII) {
assert(!U.empty());
if (FeatureDebug)
@@ -258,7 +258,7 @@
}
// Debug-only
- void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) {
+ void PrintFeatureSet(const std::vector<uint32_t> &FeatureSet) {
if (!FeatureDebug) return;
Printf("{");
for (uint32_t Feature: FeatureSet)
@@ -284,7 +284,8 @@
}
}
- void Replace(InputInfo *II, const Unit &U) {
+ void Replace(InputInfo *II, const Unit &U,
+ std::chrono::microseconds TimeOfUnit) {
assert(II->U.size() > U.size());
Hashes.erase(Sha1ToString(II->Sha1));
DeleteFile(*II);
@@ -292,6 +293,7 @@
Hashes.insert(Sha1ToString(II->Sha1));
II->U = U;
II->Reduced = true;
+ II->TimeOfUnit = TimeOfUnit;
DistributionNeedsUpdate = true;
}
@@ -325,7 +327,8 @@
const auto &II = *Inputs[i];
Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i,
Sha1ToString(II.Sha1).c_str(), II.U.size(),
- II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction);
+ II.NumExecutedMutations, II.NumSuccessfullMutations,
+ II.HasFocusFunction);
}
}
@@ -563,11 +566,11 @@
}
std::piecewise_constant_distribution<double> CorpusDistribution;
- Vector<double> Intervals;
- Vector<double> Weights;
+ std::vector<double> Intervals;
+ std::vector<double> Weights;
std::unordered_set<std::string> Hashes;
- Vector<InputInfo*> Inputs;
+ std::vector<InputInfo *> Inputs;
size_t NumAddedFeatures = 0;
size_t NumUpdatedFeatures = 0;
@@ -577,7 +580,7 @@
bool DistributionNeedsUpdate = true;
uint16_t FreqOfMostAbundantRareFeature = 0;
uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {};
- Vector<uint32_t> RareFeatures;
+ std::vector<uint32_t> RareFeatures;
std::string OutputCorpus;
};
diff --git a/libfuzzer/FuzzerDataFlowTrace.cpp b/libfuzzer/FuzzerDataFlowTrace.cpp
index 23d4225..2f9a4d2 100644
--- a/libfuzzer/FuzzerDataFlowTrace.cpp
+++ b/libfuzzer/FuzzerDataFlowTrace.cpp
@@ -37,7 +37,7 @@
// Coverage lines have this form:
// CN X Y Z T
// where N is the number of the function, T is the total number of instrumented
-// BBs, and X,Y,Z, if present, are the indecies of covered BB.
+// BBs, and X,Y,Z, if present, are the indices of covered BB.
// BB #0, which is the entry block, is not explicitly listed.
bool BlockCoverage::AppendCoverage(std::istream &IN) {
std::string L;
@@ -52,7 +52,7 @@
continue;
}
if (L[0] != 'C') continue;
- Vector<uint32_t> CoveredBlocks;
+ std::vector<uint32_t> CoveredBlocks;
while (true) {
uint32_t BB = 0;
SS >> BB;
@@ -68,7 +68,7 @@
auto It = Functions.find(FunctionId);
auto &Counters =
It == Functions.end()
- ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)})
+ ? Functions.insert({FunctionId, std::vector<uint32_t>(NumBlocks)})
.first->second
: It->second;
@@ -86,8 +86,8 @@
// * any uncovered function gets weight 0.
// * a function with lots of uncovered blocks gets bigger weight.
// * a function with a less frequently executed code gets bigger weight.
-Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
- Vector<double> Res(NumFunctions);
+std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
+ std::vector<double> Res(NumFunctions);
for (auto It : Functions) {
auto FunctionID = It.first;
auto Counters = It.second;
@@ -104,7 +104,7 @@
}
void DataFlowTrace::ReadCoverage(const std::string &DirPath) {
- Vector<SizedFile> Files;
+ std::vector<SizedFile> Files;
GetSizedFilesFromDir(DirPath, &Files);
for (auto &SF : Files) {
auto Name = Basename(SF.File);
@@ -115,16 +115,16 @@
}
}
-static void DFTStringAppendToVector(Vector<uint8_t> *DFT,
+static void DFTStringAppendToVector(std::vector<uint8_t> *DFT,
const std::string &DFTString) {
assert(DFT->size() == DFTString.size());
for (size_t I = 0, Len = DFT->size(); I < Len; I++)
(*DFT)[I] = DFTString[I] == '1';
}
-// converts a string of '0' and '1' into a Vector<uint8_t>
-static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) {
- Vector<uint8_t> DFT(DFTString.size());
+// converts a string of '0' and '1' into a std::vector<uint8_t>
+static std::vector<uint8_t> DFTStringToVector(const std::string &DFTString) {
+ std::vector<uint8_t> DFT(DFTString.size());
DFTStringAppendToVector(&DFT, DFTString);
return DFT;
}
@@ -159,14 +159,14 @@
}
bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
- Vector<SizedFile> &CorporaFiles, Random &Rand) {
+ std::vector<SizedFile> &CorporaFiles, Random &Rand) {
if (DirPath.empty()) return false;
Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str());
- Vector<SizedFile> Files;
+ std::vector<SizedFile> Files;
GetSizedFilesFromDir(DirPath, &Files);
std::string L;
size_t FocusFuncIdx = SIZE_MAX;
- Vector<std::string> FunctionNames;
+ std::vector<std::string> FunctionNames;
// Collect the hashes of the corpus files.
for (auto &SF : CorporaFiles)
@@ -191,7 +191,7 @@
// * chooses a random function according to the weights.
ReadCoverage(DirPath);
auto Weights = Coverage.FunctionWeights(NumFunctions);
- Vector<double> Intervals(NumFunctions + 1);
+ std::vector<double> Intervals(NumFunctions + 1);
std::iota(Intervals.begin(), Intervals.end(), 0);
auto Distribution = std::piecewise_constant_distribution<double>(
Intervals.begin(), Intervals.end(), Weights.begin());
@@ -247,7 +247,7 @@
}
int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
- const Vector<SizedFile> &CorporaFiles) {
+ const std::vector<SizedFile> &CorporaFiles) {
Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n",
DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size());
if (CorporaFiles.empty()) {
@@ -265,7 +265,7 @@
// we then request tags in [0,Size/2) and [Size/2, Size), and so on.
// Function number => DFT.
auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File)));
- std::unordered_map<size_t, Vector<uint8_t>> DFTMap;
+ std::unordered_map<size_t, std::vector<uint8_t>> DFTMap;
std::unordered_set<std::string> Cov;
Command Cmd;
Cmd.addArgument(DFTBinary);
diff --git a/libfuzzer/FuzzerDataFlowTrace.h b/libfuzzer/FuzzerDataFlowTrace.h
index 07c03bb..054dce1 100644
--- a/libfuzzer/FuzzerDataFlowTrace.h
+++ b/libfuzzer/FuzzerDataFlowTrace.h
@@ -39,7 +39,7 @@
namespace fuzzer {
int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
- const Vector<SizedFile> &CorporaFiles);
+ const std::vector<SizedFile> &CorporaFiles);
class BlockCoverage {
public:
@@ -77,11 +77,11 @@
return Result;
}
- Vector<double> FunctionWeights(size_t NumFunctions) const;
+ std::vector<double> FunctionWeights(size_t NumFunctions) const;
void clear() { Functions.clear(); }
private:
- typedef Vector<uint32_t> CoverageVector;
+ typedef std::vector<uint32_t> CoverageVector;
uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const {
uint32_t Res = 0;
@@ -117,9 +117,9 @@
public:
void ReadCoverage(const std::string &DirPath);
bool Init(const std::string &DirPath, std::string *FocusFunction,
- Vector<SizedFile> &CorporaFiles, Random &Rand);
+ std::vector<SizedFile> &CorporaFiles, Random &Rand);
void Clear() { Traces.clear(); }
- const Vector<uint8_t> *Get(const std::string &InputSha1) const {
+ const std::vector<uint8_t> *Get(const std::string &InputSha1) const {
auto It = Traces.find(InputSha1);
if (It != Traces.end())
return &It->second;
@@ -128,9 +128,9 @@
private:
// Input's sha1 => DFT for the FocusFunction.
- std::unordered_map<std::string, Vector<uint8_t> > Traces;
- BlockCoverage Coverage;
- std::unordered_set<std::string> CorporaHashes;
+ std::unordered_map<std::string, std::vector<uint8_t>> Traces;
+ BlockCoverage Coverage;
+ std::unordered_set<std::string> CorporaHashes;
};
} // namespace fuzzer
diff --git a/libfuzzer/FuzzerDefs.h b/libfuzzer/FuzzerDefs.h
index 1a2752a..db1f74a 100644
--- a/libfuzzer/FuzzerDefs.h
+++ b/libfuzzer/FuzzerDefs.h
@@ -38,28 +38,8 @@
// Global interface to functions that may or may not be available.
extern ExternalFunctions *EF;
-// We are using a custom allocator to give a different symbol name to STL
-// containers in order to avoid ODR violations.
-template<typename T>
- class fuzzer_allocator: public std::allocator<T> {
- public:
- fuzzer_allocator() = default;
-
- template<class U>
- fuzzer_allocator(const fuzzer_allocator<U>&) {}
-
- template<class Other>
- struct rebind { typedef fuzzer_allocator<Other> other; };
- };
-
-template<typename T>
-using Vector = std::vector<T, fuzzer_allocator<T>>;
-
-template<typename T>
-using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>;
-
-typedef Vector<uint8_t> Unit;
-typedef Vector<Unit> UnitVector;
+typedef std::vector<uint8_t> Unit;
+typedef std::vector<Unit> UnitVector;
typedef int (*UserCallback)(const uint8_t *Data, size_t Size);
int FuzzerDriver(int *argc, char ***argv, UserCallback Callback);
diff --git a/libfuzzer/FuzzerDictionary.h b/libfuzzer/FuzzerDictionary.h
index db55907..48f063c 100644
--- a/libfuzzer/FuzzerDictionary.h
+++ b/libfuzzer/FuzzerDictionary.h
@@ -52,10 +52,13 @@
public:
DictionaryEntry() {}
DictionaryEntry(Word W) : W(W) {}
- DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {}
+ DictionaryEntry(Word W, size_t PositionHint)
+ : W(W), PositionHint(PositionHint) {}
const Word &GetW() const { return W; }
- bool HasPositionHint() const { return PositionHint != std::numeric_limits<size_t>::max(); }
+ bool HasPositionHint() const {
+ return PositionHint != std::numeric_limits<size_t>::max();
+ }
size_t GetPositionHint() const {
assert(HasPositionHint());
return PositionHint;
@@ -108,12 +111,12 @@
};
// Parses one dictionary entry.
-// If successful, write the enty to Unit and returns true,
+// If successful, writes the entry to Unit and returns true,
// otherwise returns false.
bool ParseOneDictionaryEntry(const std::string &Str, Unit *U);
// Parses the dictionary file, fills Units, returns true iff all lines
// were parsed successfully.
-bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units);
+bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units);
} // namespace fuzzer
diff --git a/libfuzzer/FuzzerDriver.cpp b/libfuzzer/FuzzerDriver.cpp
index ceaa907..6b007f2 100644
--- a/libfuzzer/FuzzerDriver.cpp
+++ b/libfuzzer/FuzzerDriver.cpp
@@ -86,7 +86,7 @@
static const size_t kNumFlags =
sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]);
-static Vector<std::string> *Inputs;
+static std::vector<std::string> *Inputs;
static std::string *ProgName;
static void PrintHelp() {
@@ -187,7 +187,7 @@
}
// We don't use any library to minimize dependencies.
-static void ParseFlags(const Vector<std::string> &Args,
+static void ParseFlags(const std::vector<std::string> &Args,
const ExternalFunctions *EF) {
for (size_t F = 0; F < kNumFlags; F++) {
if (FlagDescriptions[F].IntFlag)
@@ -206,7 +206,7 @@
"Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator);
}
- Inputs = new Vector<std::string>;
+ Inputs = new std::vector<std::string>;
for (size_t A = 1; A < Args.size(); A++) {
if (ParseOneFlag(Args[A].c_str())) {
if (Flags.ignore_remaining_args)
@@ -272,7 +272,7 @@
exit(1);
}
-std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X1, const char *X2) {
std::string Cmd;
for (auto &S : Args) {
@@ -283,18 +283,19 @@
return Cmd;
}
-static int RunInMultipleProcesses(const Vector<std::string> &Args,
+static int RunInMultipleProcesses(const std::vector<std::string> &Args,
unsigned NumWorkers, unsigned NumJobs) {
std::atomic<unsigned> Counter(0);
std::atomic<bool> HasErrors(false);
Command Cmd(Args);
Cmd.removeFlag("jobs");
Cmd.removeFlag("workers");
- Vector<std::thread> V;
+ std::vector<std::thread> V;
std::thread Pulse(PulseThread);
Pulse.detach();
for (unsigned i = 0; i < NumWorkers; i++)
- V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors));
+ V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs,
+ &HasErrors));
for (auto &T : V)
T.join();
return HasErrors ? 1 : 0;
@@ -348,8 +349,8 @@
return S.substr(Beg, End - Beg);
}
-int CleanseCrashInput(const Vector<std::string> &Args,
- const FuzzingOptions &Options) {
+int CleanseCrashInput(const std::vector<std::string> &Args,
+ const FuzzingOptions &Options) {
if (Inputs->size() != 1 || !Flags.exact_artifact_path) {
Printf("ERROR: -cleanse_crash should be given one input file and"
" -exact_artifact_path\n");
@@ -372,7 +373,7 @@
auto U = FileToVector(CurrentFilePath);
size_t Size = U.size();
- const Vector<uint8_t> ReplacementBytes = {' ', 0xff};
+ const std::vector<uint8_t> ReplacementBytes = {' ', 0xff};
for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) {
bool Changed = false;
for (size_t Idx = 0; Idx < Size; Idx++) {
@@ -403,7 +404,7 @@
return 0;
}
-int MinimizeCrashInput(const Vector<std::string> &Args,
+int MinimizeCrashInput(const std::vector<std::string> &Args,
const FuzzingOptions &Options) {
if (Inputs->size() != 1) {
Printf("ERROR: -minimize_crash should be given one input file\n");
@@ -503,14 +504,15 @@
return 0;
}
-void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args,
- const Vector<std::string> &Corpora, const char *CFPathOrNull) {
+void Merge(Fuzzer *F, FuzzingOptions &Options,
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &Corpora, const char *CFPathOrNull) {
if (Corpora.size() < 2) {
Printf("INFO: Merge requires two or more corpus dirs\n");
exit(0);
}
- Vector<SizedFile> OldCorpus, NewCorpus;
+ std::vector<SizedFile> OldCorpus, NewCorpus;
GetSizedFilesFromDir(Corpora[0], &OldCorpus);
for (size_t i = 1; i < Corpora.size(); i++)
GetSizedFilesFromDir(Corpora[i], &NewCorpus);
@@ -518,10 +520,10 @@
std::sort(NewCorpus.begin(), NewCorpus.end());
std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt");
- Vector<std::string> NewFiles;
- Set<uint32_t> NewFeatures, NewCov;
+ std::vector<std::string> NewFiles;
+ std::set<uint32_t> NewFeatures, NewCov;
CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures,
- {}, &NewCov, CFPath, true);
+ {}, &NewCov, CFPath, true, Flags.set_cover_merge);
for (auto &Path : NewFiles)
F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen));
// We are done, delete the control file if it was a temporary one.
@@ -531,17 +533,17 @@
exit(0);
}
-int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict,
- UnitVector& Corpus) {
+int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict,
+ UnitVector &Corpus) {
Printf("Started dictionary minimization (up to %d tests)\n",
Dict.size() * Corpus.size() * 2);
// Scores and usage count for each dictionary unit.
- Vector<int> Scores(Dict.size());
- Vector<int> Usages(Dict.size());
+ std::vector<int> Scores(Dict.size());
+ std::vector<int> Usages(Dict.size());
- Vector<size_t> InitialFeatures;
- Vector<size_t> ModifiedFeatures;
+ std::vector<size_t> InitialFeatures;
+ std::vector<size_t> ModifiedFeatures;
for (auto &C : Corpus) {
// Get coverage for the testcase without modifications.
F->ExecuteCallback(C.data(), C.size());
@@ -551,7 +553,7 @@
});
for (size_t i = 0; i < Dict.size(); ++i) {
- Vector<uint8_t> Data = C;
+ std::vector<uint8_t> Data = C;
auto StartPos = std::search(Data.begin(), Data.end(),
Dict[i].begin(), Dict[i].end());
// Skip dictionary unit, if the testcase does not contain it.
@@ -597,9 +599,9 @@
return 0;
}
-Vector<std::string> ParseSeedInuts(const char *seed_inputs) {
+std::vector<std::string> ParseSeedInuts(const char *seed_inputs) {
// Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file
- Vector<std::string> Files;
+ std::vector<std::string> Files;
if (!seed_inputs) return Files;
std::string SeedInputs;
if (Flags.seed_inputs[0] == '@')
@@ -620,9 +622,10 @@
return Files;
}
-static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs,
- const Vector<std::string> &ExtraSeedFiles) {
- Vector<SizedFile> SizedFiles;
+static std::vector<SizedFile>
+ReadCorpora(const std::vector<std::string> &CorpusDirs,
+ const std::vector<std::string> &ExtraSeedFiles) {
+ std::vector<SizedFile> SizedFiles;
size_t LastNumFiles = 0;
for (auto &Dir : CorpusDirs) {
GetSizedFilesFromDir(Dir, &SizedFiles);
@@ -645,7 +648,7 @@
EF->LLVMFuzzerInitialize(argc, argv);
if (EF->__msan_scoped_disable_interceptor_checks)
EF->__msan_scoped_disable_interceptor_checks();
- const Vector<std::string> Args(*argv, *argv + *argc);
+ const std::vector<std::string> Args(*argv, *argv + *argc);
assert(!Args.empty());
ProgName = new std::string(Args[0]);
if (Argv0 != *ProgName) {
@@ -734,7 +737,7 @@
ValidateDirectoryExists(DirName(Options.ExactArtifactPath),
Flags.create_missing_dirs);
}
- Vector<Unit> Dictionary;
+ std::vector<Unit> Dictionary;
if (Flags.dict)
if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary))
return 1;
@@ -794,7 +797,8 @@
if (Flags.verbosity)
Printf("INFO: Seed: %u\n", Seed);
- if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) {
+ if (Flags.collect_data_flow && !Flags.fork &&
+ !(Flags.merge || Flags.set_cover_merge)) {
if (RunIndividualFiles)
return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace,
ReadCorpora({}, *Inputs));
@@ -866,10 +870,11 @@
exit(0);
}
+ Options.ForkCorpusGroups = Flags.fork_corpus_groups;
if (Flags.fork)
FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork);
- if (Flags.merge)
+ if (Flags.merge || Flags.set_cover_merge)
Merge(F, Options, Args, *Inputs, Flags.merge_control_file);
if (Flags.merge_inner) {
@@ -877,7 +882,8 @@
if (Options.MaxLen == 0)
F->SetMaxInputLen(kDefaultMaxMergeLen);
assert(Flags.merge_control_file);
- F->CrashResistantMergeInternalStep(Flags.merge_control_file);
+ F->CrashResistantMergeInternalStep(Flags.merge_control_file,
+ !strncmp(Flags.merge_inner, "2", 1));
exit(0);
}
diff --git a/libfuzzer/FuzzerExtraCounters.cpp b/libfuzzer/FuzzerExtraCounters.cpp
index 04f569a..54ecbf7 100644
--- a/libfuzzer/FuzzerExtraCounters.cpp
+++ b/libfuzzer/FuzzerExtraCounters.cpp
@@ -31,12 +31,4 @@
} // namespace fuzzer
-#else
-// TODO: implement for other platforms.
-namespace fuzzer {
-uint8_t *ExtraCountersBegin() { return nullptr; }
-uint8_t *ExtraCountersEnd() { return nullptr; }
-void ClearExtraCounters() {}
-} // namespace fuzzer
-
#endif
diff --git a/libfuzzer/FuzzerExtraCountersDarwin.cpp b/libfuzzer/FuzzerExtraCountersDarwin.cpp
new file mode 100644
index 0000000..2321ba8
--- /dev/null
+++ b/libfuzzer/FuzzerExtraCountersDarwin.cpp
@@ -0,0 +1,22 @@
+//===- FuzzerExtraCountersDarwin.cpp - Extra coverage counters for Darwin -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Extra coverage counters defined by user code for Darwin.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerPlatform.h"
+#include <cstdint>
+
+#if LIBFUZZER_APPLE
+
+namespace fuzzer {
+uint8_t *ExtraCountersBegin() { return nullptr; }
+uint8_t *ExtraCountersEnd() { return nullptr; }
+void ClearExtraCounters() {}
+} // namespace fuzzer
+
+#endif
diff --git a/libfuzzer/FuzzerExtraCountersWindows.cpp b/libfuzzer/FuzzerExtraCountersWindows.cpp
new file mode 100644
index 0000000..102f5fe
--- /dev/null
+++ b/libfuzzer/FuzzerExtraCountersWindows.cpp
@@ -0,0 +1,80 @@
+//===- FuzzerExtraCountersWindows.cpp - Extra coverage counters for Win32 -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Extra coverage counters defined by user code for Windows.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerPlatform.h"
+#include <cstdint>
+
+#if LIBFUZZER_WINDOWS
+#include <windows.h>
+
+namespace fuzzer {
+
+//
+// The __start___libfuzzer_extra_counters variable is align 16, size 16 to
+// ensure the padding between it and the next variable in this section (either
+// __libfuzzer_extra_counters or __stop___libfuzzer_extra_counters) will be
+// located at (__start___libfuzzer_extra_counters +
+// sizeof(__start___libfuzzer_extra_counters)). Otherwise, the calculation of
+// (stop - (start + sizeof(start))) might be skewed.
+//
+// The section name, __libfuzzer_extra_countaaa ends with "aaa", so it sorts
+// before __libfuzzer_extra_counters alphabetically. We want the start symbol to
+// be placed in the section just before the user supplied counters (if present).
+//
+#pragma section(".data$__libfuzzer_extra_countaaa")
+ATTRIBUTE_ALIGNED(16)
+__declspec(allocate(".data$__libfuzzer_extra_countaaa")) uint8_t
+ __start___libfuzzer_extra_counters[16] = {0};
+
+//
+// Example of what the user-supplied counters should look like. First, the
+// pragma to create the section name. It will fall alphabetically between
+// ".data$__libfuzzer_extra_countaaa" and ".data$__libfuzzer_extra_countzzz".
+// Next, the declspec to allocate the variable inside the specified section.
+// Finally, some array, struct, whatever that is used to track the counter data.
+// The size of this variable is computed at runtime by finding the difference of
+// __stop___libfuzzer_extra_counters and __start___libfuzzer_extra_counters +
+// sizeof(__start___libfuzzer_extra_counters).
+//
+
+//
+// #pragma section(".data$__libfuzzer_extra_counters")
+// __declspec(allocate(".data$__libfuzzer_extra_counters"))
+// uint8_t any_name_variable[64 * 1024];
+//
+
+//
+// Here, the section name, __libfuzzer_extra_countzzz ends with "zzz", so it
+// sorts after __libfuzzer_extra_counters alphabetically. We want the stop
+// symbol to be placed in the section just after the user supplied counters (if
+// present). Align to 1 so there isn't any padding placed between this and the
+// previous variable.
+//
+#pragma section(".data$__libfuzzer_extra_countzzz")
+ATTRIBUTE_ALIGNED(1)
+__declspec(allocate(".data$__libfuzzer_extra_countzzz")) uint8_t
+ __stop___libfuzzer_extra_counters = 0;
+
+uint8_t *ExtraCountersBegin() {
+ return __start___libfuzzer_extra_counters +
+ sizeof(__start___libfuzzer_extra_counters);
+}
+
+uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; }
+
+ATTRIBUTE_NO_SANITIZE_ALL
+void ClearExtraCounters() {
+ uint8_t *Beg = ExtraCountersBegin();
+ SecureZeroMemory(Beg, ExtraCountersEnd() - Beg);
+}
+
+} // namespace fuzzer
+
+#endif
diff --git a/libfuzzer/FuzzerFlags.def b/libfuzzer/FuzzerFlags.def
index ab31da0..1181534 100644
--- a/libfuzzer/FuzzerFlags.def
+++ b/libfuzzer/FuzzerFlags.def
@@ -58,12 +58,21 @@
FUZZER_FLAG_INT(help, 0, "Print help.")
FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens "
"in a subprocess")
+FUZZER_FLAG_INT(fork_corpus_groups, 0, "For fork mode, enable the corpus-group "
+ "strategy, The main corpus will be grouped according to size, "
+ "and each sub-process will randomly select seeds from different "
+ "groups as the sub-corpus.")
FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode")
FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode")
FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode")
FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
"merged into the 1-st corpus. Only interesting units will be taken. "
"This flag can be used to minimize a corpus.")
+FUZZER_FLAG_INT(set_cover_merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
+ "merged into the 1-st corpus. Same as the 'merge' flag, but uses the "
+ "standard greedy algorithm for the set cover problem to "
+ "compute an approximation of the minimum set of testcases that "
+ "provide the same coverage as the initial corpora")
FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists")
FUZZER_FLAG_STRING(merge_inner, "internal flag")
FUZZER_FLAG_STRING(merge_control_file,
diff --git a/libfuzzer/FuzzerFork.cpp b/libfuzzer/FuzzerFork.cpp
index 5134a5d..d59d513 100644
--- a/libfuzzer/FuzzerFork.cpp
+++ b/libfuzzer/FuzzerFork.cpp
@@ -86,18 +86,21 @@
};
struct GlobalEnv {
- Vector<std::string> Args;
- Vector<std::string> CorpusDirs;
+ std::vector<std::string> Args;
+ std::vector<std::string> CorpusDirs;
std::string MainCorpusDir;
std::string TempDir;
std::string DFTDir;
std::string DataFlowBinary;
- Set<uint32_t> Features, Cov;
- Set<std::string> FilesWithDFT;
- Vector<std::string> Files;
+ std::set<uint32_t> Features, Cov;
+ std::set<std::string> FilesWithDFT;
+ std::vector<std::string> Files;
+ std::vector<std::size_t> FilesSizes;
Random *Rand;
std::chrono::system_clock::time_point ProcessStartTime;
int Verbosity = 0;
+ int Group = 0;
+ int NumCorpuses = 8;
size_t NumTimeouts = 0;
size_t NumOOMs = 0;
@@ -136,10 +139,24 @@
if (size_t CorpusSubsetSize =
std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) {
auto Time1 = std::chrono::system_clock::now();
- for (size_t i = 0; i < CorpusSubsetSize; i++) {
- auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
- Seeds += (Seeds.empty() ? "" : ",") + SF;
- CollectDFT(SF);
+ if (Group) { // whether to group the corpus.
+ size_t AverageCorpusSize = Files.size() / NumCorpuses + 1;
+ size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize;
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ size_t RandNum = (*Rand)(AverageCorpusSize);
+ size_t Index = RandNum + StartIndex;
+ Index = Index < Files.size() ? Index
+ : Rand->SkewTowardsLast(Files.size());
+ auto &SF = Files[Index];
+ Seeds += (Seeds.empty() ? "" : ",") + SF;
+ CollectDFT(SF);
+ }
+ } else {
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
+ Seeds += (Seeds.empty() ? "" : ",") + SF;
+ CollectDFT(SF);
+ }
}
auto Time2 = std::chrono::system_clock::now();
auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
@@ -183,7 +200,7 @@
auto Stats = ParseFinalStatsFromLog(Job->LogPath);
NumRuns += Stats.number_of_executed_units;
- Vector<SizedFile> TempFiles, MergeCandidates;
+ std::vector<SizedFile> TempFiles, MergeCandidates;
// Read all newly created inputs and their feature sets.
// Choose only those inputs that have new features.
GetSizedFilesFromDir(Job->CorpusDir, &TempFiles);
@@ -193,7 +210,7 @@
FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir);
auto FeatureBytes = FileToVector(FeatureFile, 0, false);
assert((FeatureBytes.size() % sizeof(uint32_t)) == 0);
- Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
+ std::vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size());
for (auto Ft : NewFeatures) {
if (!Features.count(Ft)) {
@@ -211,15 +228,27 @@
if (MergeCandidates.empty()) return;
- Vector<std::string> FilesToAdd;
- Set<uint32_t> NewFeatures, NewCov;
+ std::vector<std::string> FilesToAdd;
+ std::set<uint32_t> NewFeatures, NewCov;
+ bool IsSetCoverMerge =
+ !Job->Cmd.getFlagValue("set_cover_merge").compare("1");
CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features,
- &NewFeatures, Cov, &NewCov, Job->CFPath, false);
+ &NewFeatures, Cov, &NewCov, Job->CFPath, false,
+ IsSetCoverMerge);
for (auto &Path : FilesToAdd) {
auto U = FileToVector(Path);
auto NewPath = DirPlusFile(MainCorpusDir, Hash(U));
WriteToFile(U, NewPath);
- Files.push_back(NewPath);
+ if (Group) { // Insert the queue according to the size of the seed.
+ size_t UnitSize = U.size();
+ auto Idx =
+ std::upper_bound(FilesSizes.begin(), FilesSizes.end(), UnitSize) -
+ FilesSizes.begin();
+ FilesSizes.insert(FilesSizes.begin() + Idx, UnitSize);
+ Files.insert(Files.begin() + Idx, NewPath);
+ } else {
+ Files.push_back(NewPath);
+ }
}
Features.insert(NewFeatures.begin(), NewFeatures.end());
Cov.insert(NewCov.begin(), NewCov.end());
@@ -228,10 +257,8 @@
if (TPC.PcIsFuncEntry(TE))
PrintPC(" NEW_FUNC: %p %F %L\n", "",
TPC.GetNextInstructionPc(TE->PC));
-
}
-
void CollectDFT(const std::string &InputPath) {
if (DataFlowBinary.empty()) return;
if (!FilesWithDFT.insert(InputPath).second) return;
@@ -283,8 +310,8 @@
// This is just a skeleton of an experimental -fork=1 feature.
void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
- const Vector<std::string> &Args,
- const Vector<std::string> &CorpusDirs, int NumJobs) {
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &CorpusDirs, int NumJobs) {
Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs);
GlobalEnv Env;
@@ -294,8 +321,9 @@
Env.Verbosity = Options.Verbosity;
Env.ProcessStartTime = std::chrono::system_clock::now();
Env.DataFlowBinary = Options.CollectDataFlow;
+ Env.Group = Options.ForkCorpusGroups;
- Vector<SizedFile> SeedFiles;
+ std::vector<SizedFile> SeedFiles;
for (auto &Dir : CorpusDirs)
GetSizedFilesFromDir(Dir, &SeedFiles);
std::sort(SeedFiles.begin(), SeedFiles.end());
@@ -316,13 +344,20 @@
Env.Files.push_back(File.File);
} else {
auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
- Set<uint32_t> NewFeatures, NewCov;
+ std::set<uint32_t> NewFeatures, NewCov;
CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features,
- &NewFeatures, Env.Cov, &NewCov, CFPath, false);
+ &NewFeatures, Env.Cov, &NewCov, CFPath,
+ /*Verbose=*/false, /*IsSetCoverMerge=*/false);
Env.Features.insert(NewFeatures.begin(), NewFeatures.end());
Env.Cov.insert(NewFeatures.begin(), NewFeatures.end());
RemoveFile(CFPath);
}
+
+ if (Env.Group) {
+ for (auto &path : Env.Files)
+ Env.FilesSizes.push_back(FileSize(path));
+ }
+
Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs,
Env.Files.size(), Env.TempDir.c_str());
@@ -337,8 +372,10 @@
WriteToFile(Unit({1}), Env.StopFile());
};
+ size_t MergeCycle = 20;
+ size_t JobExecuted = 0;
size_t JobId = 1;
- Vector<std::thread> Threads;
+ std::vector<std::thread> Threads;
for (int t = 0; t < NumJobs; t++) {
Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ));
FuzzQ.Push(Env.CreateNewJob(JobId++));
@@ -358,7 +395,46 @@
Env.RunOneMergeJob(Job.get());
- // Continue if our crash is one of the ignorred ones.
+ // merge the corpus .
+ JobExecuted++;
+ if (Env.Group && JobExecuted >= MergeCycle) {
+ std::vector<SizedFile> CurrentSeedFiles;
+ for (auto &Dir : CorpusDirs)
+ GetSizedFilesFromDir(Dir, &CurrentSeedFiles);
+ std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end());
+
+ auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
+ std::set<uint32_t> TmpNewFeatures, TmpNewCov;
+ std::set<uint32_t> TmpFeatures, TmpCov;
+ Env.Files.clear();
+ Env.FilesSizes.clear();
+ CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files,
+ TmpFeatures, &TmpNewFeatures, TmpCov, &TmpNewCov,
+ CFPath, /*Verbose=*/false, /*IsSetCoverMerge=*/false);
+ for (auto &path : Env.Files)
+ Env.FilesSizes.push_back(FileSize(path));
+ RemoveFile(CFPath);
+ JobExecuted = 0;
+ MergeCycle += 5;
+ }
+
+ // Since the number of corpus seeds will gradually increase, in order to
+ // control the number in each group to be about three times the number of
+ // seeds selected each time, the number of groups is dynamically adjusted.
+ if (Env.Files.size() < 2000)
+ Env.NumCorpuses = 12;
+ else if (Env.Files.size() < 6000)
+ Env.NumCorpuses = 20;
+ else if (Env.Files.size() < 12000)
+ Env.NumCorpuses = 32;
+ else if (Env.Files.size() < 16000)
+ Env.NumCorpuses = 40;
+ else if (Env.Files.size() < 24000)
+ Env.NumCorpuses = 60;
+ else
+ Env.NumCorpuses = 80;
+
+ // Continue if our crash is one of the ignored ones.
if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
Env.NumTimeouts++;
else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
diff --git a/libfuzzer/FuzzerFork.h b/libfuzzer/FuzzerFork.h
index b29a43e..fc3e9d6 100644
--- a/libfuzzer/FuzzerFork.h
+++ b/libfuzzer/FuzzerFork.h
@@ -17,8 +17,8 @@
namespace fuzzer {
void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
- const Vector<std::string> &Args,
- const Vector<std::string> &CorpusDirs, int NumJobs);
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &CorpusDirs, int NumJobs);
} // namespace fuzzer
#endif // LLVM_FUZZER_FORK_H
diff --git a/libfuzzer/FuzzerIO.cpp b/libfuzzer/FuzzerIO.cpp
index 7f149ac..0a58c53 100644
--- a/libfuzzer/FuzzerIO.cpp
+++ b/libfuzzer/FuzzerIO.cpp
@@ -23,6 +23,14 @@
static FILE *OutputFile = stderr;
+FILE *GetOutputFile() {
+ return OutputFile;
+}
+
+void SetOutputFile(FILE *NewOutputFile) {
+ OutputFile = NewOutputFile;
+}
+
long GetEpoch(const std::string &Path) {
struct stat St;
if (stat(Path.c_str(), &St))
@@ -90,11 +98,11 @@
fclose(Out);
}
-void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch,
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch,
size_t MaxSize, bool ExitOnError,
- Vector<std::string> *VPaths) {
+ std::vector<std::string> *VPaths) {
long E = Epoch ? *Epoch : 0;
- Vector<std::string> Files;
+ std::vector<std::string> Files;
ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true);
size_t NumLoaded = 0;
for (size_t i = 0; i < Files.size(); i++) {
@@ -112,8 +120,8 @@
}
}
-void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) {
- Vector<std::string> Files;
+void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V) {
+ std::vector<std::string> Files;
ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true);
for (auto &File : Files)
if (size_t Size = FileSize(File))
diff --git a/libfuzzer/FuzzerIO.h b/libfuzzer/FuzzerIO.h
index bde1826..401afa0 100644
--- a/libfuzzer/FuzzerIO.h
+++ b/libfuzzer/FuzzerIO.h
@@ -32,9 +32,9 @@
void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path);
void AppendToFile(const std::string &Data, const std::string &Path);
-void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch,
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch,
size_t MaxSize, bool ExitOnError,
- Vector<std::string> *VPaths = 0);
+ std::vector<std::string> *VPaths = 0);
// Returns "Dir/FileName" or equivalent for the current OS.
std::string DirPlusFile(const std::string &DirPath,
@@ -54,6 +54,10 @@
void CloseStdout();
+// For testing.
+FILE *GetOutputFile();
+void SetOutputFile(FILE *NewOutputFile);
+
void Printf(const char *Fmt, ...);
void VPrintf(bool Verbose, const char *Fmt, ...);
@@ -66,7 +70,7 @@
size_t FileSize(const std::string &Path);
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir);
+ std::vector<std::string> *V, bool TopDir);
bool MkDirRecursive(const std::string &Dir);
void RmDirRecursive(const std::string &Dir);
@@ -85,7 +89,7 @@
bool operator<(const SizedFile &B) const { return Size < B.Size; }
};
-void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V);
+void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V);
char GetSeparator();
bool IsSeparator(char C);
diff --git a/libfuzzer/FuzzerIOPosix.cpp b/libfuzzer/FuzzerIOPosix.cpp
index 4706a40..3700fb0 100644
--- a/libfuzzer/FuzzerIOPosix.cpp
+++ b/libfuzzer/FuzzerIOPosix.cpp
@@ -53,7 +53,7 @@
}
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir) {
+ std::vector<std::string> *V, bool TopDir) {
auto E = GetEpoch(Dir);
if (Epoch)
if (E && *Epoch >= E) return;
@@ -78,7 +78,6 @@
*Epoch = E;
}
-
void IterateDirRecursive(const std::string &Dir,
void (*DirPreCallback)(const std::string &Dir),
void (*DirPostCallback)(const std::string &Dir),
diff --git a/libfuzzer/FuzzerIOWindows.cpp b/libfuzzer/FuzzerIOWindows.cpp
index 61ad35e..6771fc1 100644
--- a/libfuzzer/FuzzerIOWindows.cpp
+++ b/libfuzzer/FuzzerIOWindows.cpp
@@ -111,7 +111,7 @@
}
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir) {
+ std::vector<std::string> *V, bool TopDir) {
auto E = GetEpoch(Dir);
if (Epoch)
if (E && *Epoch >= E) return;
@@ -159,7 +159,6 @@
*Epoch = E;
}
-
void IterateDirRecursive(const std::string &Dir,
void (*DirPreCallback)(const std::string &Dir),
void (*DirPostCallback)(const std::string &Dir),
@@ -297,9 +296,8 @@
return Pos - Offset;
}
-// Parse the given Ref string from the position Offset, to exactly match the given
-// string Patt.
-// Returns number of characters considered if successful.
+// Parse the given Ref string from the position Offset, to exactly match the
+// given string Patt. Returns number of characters considered if successful.
static size_t ParseCustomString(const std::string &Ref, size_t Offset,
const char *Patt) {
size_t Len = strlen(Patt);
diff --git a/libfuzzer/FuzzerInterceptors.cpp b/libfuzzer/FuzzerInterceptors.cpp
index b877986..d5b0a42 100644
--- a/libfuzzer/FuzzerInterceptors.cpp
+++ b/libfuzzer/FuzzerInterceptors.cpp
@@ -25,6 +25,7 @@
}
#include <cassert>
+#include <cstddef> // for size_t
#include <cstdint>
#include <dlfcn.h> // for dlsym()
diff --git a/libfuzzer/FuzzerInternal.h b/libfuzzer/FuzzerInternal.h
index 37c8a01..31f54ea 100644
--- a/libfuzzer/FuzzerInternal.h
+++ b/libfuzzer/FuzzerInternal.h
@@ -35,8 +35,8 @@
Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
FuzzingOptions Options);
~Fuzzer();
- void Loop(Vector<SizedFile> &CorporaFiles);
- void ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles);
+ void Loop(std::vector<SizedFile> &CorporaFiles);
+ void ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles);
void MinimizeCrashLoop(const Unit &U);
void RereadOutputCorpus(size_t MaxSize);
@@ -65,15 +65,19 @@
static void StaticFileSizeExceedCallback();
static void StaticGracefulExitCallback();
- void ExecuteCallback(const uint8_t *Data, size_t Size);
+ // Executes the target callback on {Data, Size} once.
+ // Returns false if the input was rejected by the target (target returned -1),
+ // and true otherwise.
+ bool ExecuteCallback(const uint8_t *Data, size_t Size);
bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false,
InputInfo *II = nullptr, bool ForceAddToCorpus = false,
bool *FoundUniqFeatures = nullptr);
void TPCUpdateObservedPCs();
// Merge Corpora[1:] into Corpora[0].
- void Merge(const Vector<std::string> &Corpora);
- void CrashResistantMergeInternalStep(const std::string &ControlFilePath);
+ void Merge(const std::vector<std::string> &Corpora);
+ void CrashResistantMergeInternalStep(const std::string &ControlFilePath,
+ bool IsSetCoverMerge);
MutationDispatcher &GetMD() { return MD; }
void PrintFinalStats();
void SetMaxInputLen(size_t MaxInputLen);
@@ -141,7 +145,7 @@
size_t MaxMutationLen = 0;
size_t TmpMaxMutationLen = 0;
- Vector<uint32_t> UniqFeatureSetTmp;
+ std::vector<uint32_t> UniqFeatureSetTmp;
// Need to know our own thread.
static thread_local bool IsMyThread;
diff --git a/libfuzzer/FuzzerLoop.cpp b/libfuzzer/FuzzerLoop.cpp
index 86a78ab..f095757 100644
--- a/libfuzzer/FuzzerLoop.cpp
+++ b/libfuzzer/FuzzerLoop.cpp
@@ -388,7 +388,7 @@
void Fuzzer::CheckExitOnSrcPosOrItem() {
if (!Options.ExitOnSrcPos.empty()) {
- static auto *PCsSet = new Set<uintptr_t>;
+ static auto *PCsSet = new std::set<uintptr_t>;
auto HandlePC = [&](const TracePC::PCTableEntry *TE) {
if (!PCsSet->insert(TE->PC).second)
return;
@@ -413,8 +413,8 @@
void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec)
return;
- Vector<Unit> AdditionalCorpus;
- Vector<std::string> AdditionalCorpusPaths;
+ std::vector<Unit> AdditionalCorpus;
+ std::vector<std::string> AdditionalCorpusPaths;
ReadDirToVectorOfUnits(
Options.OutputCorpus.c_str(), &AdditionalCorpus,
&EpochOfLastReadOfOutputCorpus, MaxSize,
@@ -457,7 +457,7 @@
static void WriteFeatureSetToFile(const std::string &FeaturesDir,
const std::string &FileName,
- const Vector<uint32_t> &FeatureSet) {
+ const std::vector<uint32_t> &FeatureSet) {
if (FeaturesDir.empty() || FeatureSet.empty()) return;
WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()),
FeatureSet.size() * sizeof(FeatureSet[0]),
@@ -511,7 +511,7 @@
// Largest input length should be INT_MAX.
assert(Size < std::numeric_limits<uint32_t>::max());
- ExecuteCallback(Data, Size);
+ if(!ExecuteCallback(Data, Size)) return false;
auto TimeOfUnit = duration_cast<microseconds>(UnitStopTime - UnitStartTime);
UniqFeatureSetTmp.clear();
@@ -548,7 +548,7 @@
FoundUniqFeaturesOfII == II->UniqFeatureSet.size() &&
II->U.size() > Size) {
auto OldFeaturesFile = Sha1ToString(II->Sha1);
- Corpus.Replace(II, {Data, Data + Size});
+ Corpus.Replace(II, {Data, Data + Size}, TimeOfUnit);
RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile,
Sha1ToString(II->Sha1));
return true;
@@ -586,7 +586,7 @@
// This method is not inlined because it would cause a test to fail where it
// is part of the stack unwinding. See D97975 for details.
-ATTRIBUTE_NOINLINE void Fuzzer::ExecuteCallback(const uint8_t *Data,
+ATTRIBUTE_NOINLINE bool Fuzzer::ExecuteCallback(const uint8_t *Data,
size_t Size) {
TPC.RecordInitialStack();
TotalNumberOfRuns++;
@@ -602,23 +602,24 @@
if (CurrentUnitData && CurrentUnitData != Data)
memcpy(CurrentUnitData, Data, Size);
CurrentUnitSize = Size;
+ int CBRes = 0;
{
ScopedEnableMsanInterceptorChecks S;
AllocTracer.Start(Options.TraceMalloc);
UnitStartTime = system_clock::now();
TPC.ResetMaps();
RunningUserCallback = true;
- int Res = CB(DataCopy, Size);
+ CBRes = CB(DataCopy, Size);
RunningUserCallback = false;
UnitStopTime = system_clock::now();
- (void)Res;
- assert(Res == 0);
+ assert(CBRes == 0 || CBRes == -1);
HasMoreMallocsThanFrees = AllocTracer.Stop();
}
if (!LooseMemeq(DataCopy, Data, Size))
CrashOnOverwrittenData();
CurrentUnitSize = 0;
delete[] DataCopy;
+ return CBRes == 0;
}
std::string Fuzzer::WriteToOutputCorpus(const Unit &U) {
@@ -784,7 +785,7 @@
LastAllocatorPurgeAttemptTime = system_clock::now();
}
-void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) {
+void Fuzzer::ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles) {
const size_t kMaxSaneLen = 1 << 20;
const size_t kMinDefaultLen = 4096;
size_t MaxSize = 0;
@@ -843,13 +844,20 @@
}
if (Corpus.empty() && Options.MaxNumberOfRuns) {
- Printf("ERROR: no interesting inputs were found. "
- "Is the code instrumented for coverage? Exiting.\n");
- exit(1);
+ Printf("WARNING: no interesting inputs were found so far. "
+ "Is the code instrumented for coverage?\n"
+ "This may also happen if the target rejected all inputs we tried so "
+ "far\n");
+ // The remaining logic requires that the corpus is not empty,
+ // so we add one fake input to the in-memory corpus.
+ Corpus.AddToCorpus({'\n'}, /*NumFeatures=*/1, /*MayDeleteFile=*/true,
+ /*HasFocusFunction=*/false, /*NeverReduce=*/false,
+ /*TimeOfUnit=*/duration_cast<microseconds>(0s), {0}, DFT,
+ /*BaseII*/ nullptr);
}
}
-void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) {
+void Fuzzer::Loop(std::vector<SizedFile> &CorporaFiles) {
auto FocusFunctionOrAuto = Options.FocusFunction;
DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles,
MD.GetRand());
diff --git a/libfuzzer/FuzzerMerge.cpp b/libfuzzer/FuzzerMerge.cpp
index 162453c..24bd119 100644
--- a/libfuzzer/FuzzerMerge.cpp
+++ b/libfuzzer/FuzzerMerge.cpp
@@ -77,8 +77,8 @@
size_t ExpectedStartMarker = 0;
const size_t kInvalidStartMarker = -1;
size_t LastSeenStartMarker = kInvalidStartMarker;
- Vector<uint32_t> TmpFeatures;
- Set<uint32_t> PCs;
+ std::vector<uint32_t> TmpFeatures;
+ std::set<uint32_t> PCs;
while (std::getline(IS, Line, '\n')) {
std::istringstream ISS1(Line);
std::string Marker;
@@ -132,15 +132,16 @@
// Decides which files need to be merged (add those to NewFiles).
// Returns the number of new features added.
-size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov,
- Vector<std::string> *NewFiles) {
+size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles) {
NewFiles->clear();
NewFeatures->clear();
NewCov->clear();
assert(NumFilesInFirstCorpus <= Files.size());
- Set<uint32_t> AllFeatures = InitialFeatures;
+ std::set<uint32_t> AllFeatures = InitialFeatures;
// What features are in the initial corpus?
for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
@@ -150,7 +151,7 @@
// Remove all features that we already know from all other inputs.
for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
auto &Cur = Files[i].Features;
- Vector<uint32_t> Tmp;
+ std::vector<uint32_t> Tmp;
std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(),
AllFeatures.end(), std::inserter(Tmp, Tmp.begin()));
Cur.swap(Tmp);
@@ -188,15 +189,16 @@
return NewFeatures->size();
}
-Set<uint32_t> Merger::AllFeatures() const {
- Set<uint32_t> S;
+std::set<uint32_t> Merger::AllFeatures() const {
+ std::set<uint32_t> S;
for (auto &File : Files)
S.insert(File.Features.begin(), File.Features.end());
return S;
}
// Inner process. May crash if the target crashes.
-void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
+void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath,
+ bool IsSetCoverMerge) {
Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str());
Merger M;
std::ifstream IF(CFPath);
@@ -212,11 +214,11 @@
M.Files.size() - M.FirstNotProcessedFile);
std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app);
- Set<size_t> AllFeatures;
+ std::set<size_t> AllFeatures;
auto PrintStatsWrapper = [this, &AllFeatures](const char* Where) {
this->PrintStats(Where, "\n", 0, AllFeatures.size());
};
- Set<const TracePC::PCTableEntry *> AllPCs;
+ std::set<const TracePC::PCTableEntry *> AllPCs;
for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) {
Fuzzer::MaybeExitGracefully();
auto U = FileToVector(M.Files[i].Name);
@@ -234,13 +236,14 @@
// Collect coverage. We are iterating over the files in this order:
// * First, files in the initial corpus ordered by size, smallest first.
// * Then, all other files, smallest first.
- // So it makes no sense to record all features for all files, instead we
- // only record features that were not seen before.
- Set<size_t> UniqFeatures;
- TPC.CollectFeatures([&](size_t Feature) {
- if (AllFeatures.insert(Feature).second)
- UniqFeatures.insert(Feature);
- });
+ std::set<size_t> Features;
+ if (IsSetCoverMerge)
+ TPC.CollectFeatures([&](size_t Feature) { Features.insert(Feature); });
+ else
+ TPC.CollectFeatures([&](size_t Feature) {
+ if (AllFeatures.insert(Feature).second)
+ Features.insert(Feature);
+ });
TPC.UpdateObservedPCs();
// Show stats.
if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)))
@@ -249,7 +252,7 @@
PrintStatsWrapper("LOADED");
// Write the post-run marker and the coverage.
OF << "FT " << i;
- for (size_t F : UniqFeatures)
+ for (size_t F : Features)
OF << " " << F;
OF << "\n";
OF << "COV " << i;
@@ -263,15 +266,137 @@
PrintStatsWrapper("DONE ");
}
-static size_t WriteNewControlFile(const std::string &CFPath,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- const Vector<MergeFileInfo> &KnownFiles) {
+// Merges all corpora into the first corpus. A file is added into
+// the first corpus only if it adds new features. Unlike `Merger::Merge`,
+// this implementation calculates an approximation of the minimum set
+// of corpora files, that cover all known features (set cover problem).
+// Generally, this means that files with more features are preferred for
+// merge into the first corpus. When two files have the same number of
+// features, the smaller one is preferred.
+size_t Merger::SetCoverMerge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles) {
+ assert(NumFilesInFirstCorpus <= Files.size());
+ NewFiles->clear();
+ NewFeatures->clear();
+ NewCov->clear();
+ std::set<uint32_t> AllFeatures;
+ // 1 << 21 - 1 is the maximum feature index.
+ // See 'kFeatureSetSize' in 'FuzzerCorpus.h'.
+ const uint32_t kFeatureSetSize = 1 << 21;
+ std::vector<bool> Covered(kFeatureSetSize, false);
+ size_t NumCovered = 0;
+
+ std::set<uint32_t> ExistingFeatures = InitialFeatures;
+ for (size_t i = 0; i < NumFilesInFirstCorpus; ++i)
+ ExistingFeatures.insert(Files[i].Features.begin(), Files[i].Features.end());
+
+ // Mark the existing features as covered.
+ for (const auto &F : ExistingFeatures) {
+ if (!Covered[F % kFeatureSetSize]) {
+ ++NumCovered;
+ Covered[F % kFeatureSetSize] = true;
+ }
+ // Calculate an underestimation of the set of covered features
+ // since the `Covered` bitvector is smaller than the feature range.
+ AllFeatures.insert(F % kFeatureSetSize);
+ }
+
+ std::set<size_t> RemainingFiles;
+ for (size_t i = NumFilesInFirstCorpus; i < Files.size(); ++i) {
+ // Construct an incremental sequence which represent the
+ // indices to all files (excluding those in the initial corpus).
+ // RemainingFiles = range(NumFilesInFirstCorpus..Files.size()).
+ RemainingFiles.insert(i);
+ // Insert this file's unique features to all features.
+ for (const auto &F : Files[i].Features)
+ AllFeatures.insert(F % kFeatureSetSize);
+ }
+
+ // Integrate files into Covered until set is complete.
+ while (NumCovered != AllFeatures.size()) {
+ // Index to file with largest number of unique features.
+ size_t MaxFeaturesIndex = NumFilesInFirstCorpus;
+ // Indices to remove from RemainingFiles.
+ std::set<size_t> RemoveIndices;
+ // Running max unique feature count.
+ // Updated upon finding a file with more features.
+ size_t MaxNumFeatures = 0;
+
+ // Iterate over all files not yet integrated into Covered,
+ // to find the file which has the largest number of
+ // features that are not already in Covered.
+ for (const auto &i : RemainingFiles) {
+ const auto &File = Files[i];
+ size_t CurrentUnique = 0;
+ // Count number of features in this file
+ // which are not yet in Covered.
+ for (const auto &F : File.Features)
+ if (!Covered[F % kFeatureSetSize])
+ ++CurrentUnique;
+
+ if (CurrentUnique == 0) {
+ // All features in this file are already in Covered: skip next time.
+ RemoveIndices.insert(i);
+ } else if (CurrentUnique > MaxNumFeatures ||
+ (CurrentUnique == MaxNumFeatures &&
+ File.Size < Files[MaxFeaturesIndex].Size)) {
+ // Update the max features file based on unique features
+ // Break ties by selecting smaller files.
+ MaxNumFeatures = CurrentUnique;
+ MaxFeaturesIndex = i;
+ }
+ }
+ // Must be a valid index/
+ assert(MaxFeaturesIndex < Files.size());
+ // Remove any feature-less files found.
+ for (const auto &i : RemoveIndices)
+ RemainingFiles.erase(i);
+ if (MaxNumFeatures == 0) {
+ // Did not find a file that adds unique features.
+ // This means that we should have no remaining files.
+ assert(RemainingFiles.size() == 0);
+ assert(NumCovered == AllFeatures.size());
+ break;
+ }
+
+ // MaxFeaturesIndex must be an element of Remaining.
+ assert(RemainingFiles.find(MaxFeaturesIndex) != RemainingFiles.end());
+ // Remove the file with the most features from Remaining.
+ RemainingFiles.erase(MaxFeaturesIndex);
+ const auto &MaxFeatureFile = Files[MaxFeaturesIndex];
+ // Add the features of the max feature file to Covered.
+ for (const auto &F : MaxFeatureFile.Features) {
+ if (!Covered[F % kFeatureSetSize]) {
+ ++NumCovered;
+ Covered[F % kFeatureSetSize] = true;
+ NewFeatures->insert(F);
+ }
+ }
+ // Add the index to this file to the result.
+ NewFiles->push_back(MaxFeatureFile.Name);
+ // Update NewCov with the additional coverage
+ // that MaxFeatureFile provides.
+ for (const auto &C : MaxFeatureFile.Cov)
+ if (InitialCov.find(C) == InitialCov.end())
+ NewCov->insert(C);
+ }
+
+ return NewFeatures->size();
+}
+
+static size_t
+WriteNewControlFile(const std::string &CFPath,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ const std::vector<MergeFileInfo> &KnownFiles) {
std::unordered_set<std::string> FilesToSkip;
for (auto &SF: KnownFiles)
FilesToSkip.insert(SF.Name);
- Vector<std::string> FilesToUse;
+ std::vector<std::string> FilesToUse;
auto MaybeUseFile = [=, &FilesToUse](std::string Name) {
if (FilesToSkip.find(Name) == FilesToSkip.end())
FilesToUse.push_back(Name);
@@ -299,19 +424,19 @@
}
// Outer process. Does not call the target code and thus should not fail.
-void CrashResistantMerge(const Vector<std::string> &Args,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- Vector<std::string> *NewFiles,
- const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov,
- Set<uint32_t> *NewCov,
- const std::string &CFPath,
- bool V /*Verbose*/) {
+void CrashResistantMerge(const std::vector<std::string> &Args,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ std::vector<std::string> *NewFiles,
+ const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov, const std::string &CFPath,
+ bool V, /*Verbose*/
+ bool IsSetCoverMerge) {
if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge.
size_t NumAttempts = 0;
- Vector<MergeFileInfo> KnownFiles;
+ std::vector<MergeFileInfo> KnownFiles;
if (FileSize(CFPath)) {
VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n",
CFPath.c_str());
@@ -363,6 +488,7 @@
// Every inner process should execute at least one input.
Command BaseCmd(Args);
BaseCmd.removeFlag("merge");
+ BaseCmd.removeFlag("set_cover_merge");
BaseCmd.removeFlag("fork");
BaseCmd.removeFlag("collect_data_flow");
for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) {
@@ -370,14 +496,16 @@
VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt);
Command Cmd(BaseCmd);
Cmd.addFlag("merge_control_file", CFPath);
- Cmd.addFlag("merge_inner", "1");
+ // If we are going to use the set cover implementation for
+ // minimization add the merge_inner=2 internal flag.
+ Cmd.addFlag("merge_inner", IsSetCoverMerge ? "2" : "1");
if (!V) {
Cmd.setOutputFile(getDevNull());
Cmd.combineOutAndErr();
}
auto ExitCode = ExecuteCommand(Cmd);
if (!ExitCode) {
- VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt);
+ VPrintf(V, "MERGE-OUTER: successful in %zd attempt(s)\n", Attempt);
break;
}
}
@@ -395,7 +523,10 @@
M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb());
M.Files.insert(M.Files.end(), KnownFiles.begin(), KnownFiles.end());
- M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
+ if (IsSetCoverMerge)
+ M.SetCoverMerge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
+ else
+ M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; "
"%zd new coverage edges\n",
NewFiles->size(), NewFeatures->size(), NewCov->size());
diff --git a/libfuzzer/FuzzerMerge.h b/libfuzzer/FuzzerMerge.h
index e0c6bc5..42f798e 100644
--- a/libfuzzer/FuzzerMerge.h
+++ b/libfuzzer/FuzzerMerge.h
@@ -41,6 +41,7 @@
#define LLVM_FUZZER_MERGE_H
#include "FuzzerDefs.h"
+#include "FuzzerIO.h"
#include <istream>
#include <ostream>
@@ -52,11 +53,11 @@
struct MergeFileInfo {
std::string Name;
size_t Size = 0;
- Vector<uint32_t> Features, Cov;
+ std::vector<uint32_t> Features, Cov;
};
struct Merger {
- Vector<MergeFileInfo> Files;
+ std::vector<MergeFileInfo> Files;
size_t NumFilesInFirstCorpus = 0;
size_t FirstNotProcessedFile = 0;
std::string LastFailure;
@@ -64,23 +65,28 @@
bool Parse(std::istream &IS, bool ParseCoverage);
bool Parse(const std::string &Str, bool ParseCoverage);
void ParseOrExit(std::istream &IS, bool ParseCoverage);
- size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov,
- Vector<std::string> *NewFiles);
+ size_t Merge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov, std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles);
+ size_t SetCoverMerge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles);
size_t ApproximateMemoryConsumption() const;
- Set<uint32_t> AllFeatures() const;
+ std::set<uint32_t> AllFeatures() const;
};
-void CrashResistantMerge(const Vector<std::string> &Args,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- Vector<std::string> *NewFiles,
- const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov,
- Set<uint32_t> *NewCov,
- const std::string &CFPath,
- bool Verbose);
+void CrashResistantMerge(const std::vector<std::string> &Args,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ std::vector<std::string> *NewFiles,
+ const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov, const std::string &CFPath,
+ bool Verbose, bool IsSetCoverMerge);
} // namespace fuzzer
diff --git a/libfuzzer/FuzzerMutate.cpp b/libfuzzer/FuzzerMutate.cpp
index 4650f1b..d663900 100644
--- a/libfuzzer/FuzzerMutate.cpp
+++ b/libfuzzer/FuzzerMutate.cpp
@@ -485,7 +485,7 @@
}
void MutationDispatcher::PrintRecommendedDictionary() {
- Vector<DictionaryEntry> V;
+ std::vector<DictionaryEntry> V;
for (auto &DE : PersistentAutoDictionary)
if (!ManualDictionary.ContainsWord(DE.GetW()))
V.push_back(DE);
@@ -540,7 +540,7 @@
// Mutates Data in place, returns new size.
size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
size_t MaxSize,
- Vector<Mutator> &Mutators) {
+ std::vector<Mutator> &Mutators) {
assert(MaxSize > 0);
// Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize),
// in which case they will return 0.
@@ -562,7 +562,7 @@
// Mask represents the set of Data bytes that are worth mutating.
size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size,
size_t MaxSize,
- const Vector<uint8_t> &Mask) {
+ const std::vector<uint8_t> &Mask) {
size_t MaskedSize = std::min(Size, Mask.size());
// * Copy the worthy bytes into a temporary array T
// * Mutate T
diff --git a/libfuzzer/FuzzerMutate.h b/libfuzzer/FuzzerMutate.h
index fd37191..97704e2 100644
--- a/libfuzzer/FuzzerMutate.h
+++ b/libfuzzer/FuzzerMutate.h
@@ -77,7 +77,7 @@
/// that have '1' in Mask.
/// Mask.size() should be >= Size.
size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize,
- const Vector<uint8_t> &Mask);
+ const std::vector<uint8_t> &Mask);
/// Applies one of the default mutations. Provided as a service
/// to mutation authors.
@@ -104,7 +104,7 @@
size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
size_t MaxSize);
size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize,
- Vector<Mutator> &Mutators);
+ std::vector<Mutator> &Mutators);
size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To,
size_t ToSize, size_t MaxToSize);
@@ -133,22 +133,22 @@
// entries that led to successful discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
- Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+ std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
static const size_t kCmpDictionaryEntriesDequeSize = 16;
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
size_t CmpDictionaryEntriesDequeIdx = 0;
const Unit *CrossOverWith = nullptr;
- Vector<uint8_t> MutateInPlaceHere;
- Vector<uint8_t> MutateWithMaskTemp;
+ std::vector<uint8_t> MutateInPlaceHere;
+ std::vector<uint8_t> MutateWithMaskTemp;
// CustomCrossOver needs its own buffer as a custom implementation may call
// LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere.
- Vector<uint8_t> CustomCrossOverInPlaceHere;
+ std::vector<uint8_t> CustomCrossOverInPlaceHere;
- Vector<Mutator> Mutators;
- Vector<Mutator> DefaultMutators;
- Vector<Mutator> CurrentMutatorSequence;
+ std::vector<Mutator> Mutators;
+ std::vector<Mutator> DefaultMutators;
+ std::vector<Mutator> CurrentMutatorSequence;
};
} // namespace fuzzer
diff --git a/libfuzzer/FuzzerOptions.h b/libfuzzer/FuzzerOptions.h
index d0c285a..72e2561 100644
--- a/libfuzzer/FuzzerOptions.h
+++ b/libfuzzer/FuzzerOptions.h
@@ -47,6 +47,7 @@
int ReportSlowUnits = 10;
bool OnlyASCII = false;
bool Entropic = true;
+ bool ForkCorpusGroups = false;
size_t EntropicFeatureFrequencyThreshold = 0xFF;
size_t EntropicNumberOfRarestFeatures = 100;
bool EntropicScalePerExecTime = false;
diff --git a/libfuzzer/FuzzerTracePC.cpp b/libfuzzer/FuzzerTracePC.cpp
index d808b9b..f12f7aa 100644
--- a/libfuzzer/FuzzerTracePC.cpp
+++ b/libfuzzer/FuzzerTracePC.cpp
@@ -133,13 +133,14 @@
// so we return (pc-2) in that case in order to be safe.
// For A32 mode we return (pc-4) because all instructions are 32 bit long.
return (PC - 3) & (~1);
-#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__)
- // PCs are always 4 byte aligned.
- return PC - 4;
#elif defined(__sparc__) || defined(__mips__)
return PC - 8;
-#else
+#elif defined(__riscv__)
+ return PC - 2;
+#elif defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)
return PC - 1;
+#else
+ return PC - 4;
#endif
}
@@ -157,7 +158,7 @@
}
void TracePC::UpdateObservedPCs() {
- Vector<uintptr_t> CoveredFuncs;
+ std::vector<uintptr_t> CoveredFuncs;
auto ObservePC = [&](const PCTableEntry *TE) {
if (ObservedPCs.insert(TE).second && DoPrintNewPCs) {
PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p",
@@ -300,8 +301,8 @@
FunctionStr = FunctionStr.substr(3);
std::string LineStr = DescribePC("%l", VisualizePC);
size_t NumEdges = Last - First;
- Vector<uintptr_t> UncoveredPCs;
- Vector<uintptr_t> CoveredPCs;
+ std::vector<uintptr_t> UncoveredPCs;
+ std::vector<uintptr_t> CoveredPCs;
for (auto TE = First; TE < Last; TE++)
if (!ObservedPCs.count(TE))
UncoveredPCs.push_back(TE->PC);
@@ -391,6 +392,7 @@
ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance);
}
+ATTRIBUTE_NO_SANITIZE_MEMORY
static size_t InternalStrnlen(const char *S, size_t MaxLen) {
size_t Len = 0;
for (; Len < MaxLen && S[Len]; Len++) {}
@@ -398,7 +400,8 @@
}
// Finds min of (strlen(S1), strlen(S2)).
-// Needed bacause one of these strings may actually be non-zero terminated.
+// Needed because one of these strings may actually be non-zero terminated.
+ATTRIBUTE_NO_SANITIZE_MEMORY
static size_t InternalStrnlen2(const char *S1, const char *S2) {
size_t Len = 0;
for (; S1[Len] && S2[Len]; Len++) {}
diff --git a/libfuzzer/FuzzerTracePC.h b/libfuzzer/FuzzerTracePC.h
index a937329..af1f9d8 100644
--- a/libfuzzer/FuzzerTracePC.h
+++ b/libfuzzer/FuzzerTracePC.h
@@ -169,7 +169,7 @@
size_t NumPCTables;
size_t NumPCsInPCTables;
- Set<const PCTableEntry*> ObservedPCs;
+ std::set<const PCTableEntry *> ObservedPCs;
std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter.
uint8_t *FocusFunctionCounterPtr = nullptr;
diff --git a/libfuzzer/FuzzerUtil.cpp b/libfuzzer/FuzzerUtil.cpp
index 0518549..aeab70f 100644
--- a/libfuzzer/FuzzerUtil.cpp
+++ b/libfuzzer/FuzzerUtil.cpp
@@ -43,7 +43,7 @@
else if (Byte >= 32 && Byte < 127)
Printf("%c", Byte);
else
- Printf("\\x%02x", Byte);
+ Printf("\\%03o", Byte);
}
void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) {
@@ -124,7 +124,7 @@
return true;
}
-bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) {
+bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units) {
if (Text.empty()) {
Printf("ParseDictionaryFile: file does not exist or is empty\n");
return false;
diff --git a/libfuzzer/FuzzerUtil.h b/libfuzzer/FuzzerUtil.h
index a188a7b..71d4909 100644
--- a/libfuzzer/FuzzerUtil.h
+++ b/libfuzzer/FuzzerUtil.h
@@ -66,10 +66,10 @@
const void *SearchMemory(const void *haystack, size_t haystacklen,
const void *needle, size_t needlelen);
-std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X1, const char *X2);
-inline std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+inline std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X) {
return CloneArgsWithoutX(Args, X, X);
}
diff --git a/libfuzzer/FuzzerUtilFuchsia.cpp b/libfuzzer/FuzzerUtilFuchsia.cpp
index 5034b4a..d80b80c 100644
--- a/libfuzzer/FuzzerUtilFuchsia.cpp
+++ b/libfuzzer/FuzzerUtilFuchsia.cpp
@@ -52,6 +52,12 @@
namespace {
+// The signal handler thread uses Zircon exceptions to resume crashed threads
+// into libFuzzer's POSIX signal handlers. The associated event is used to
+// signal when the thread is running, and when it should stop.
+std::thread SignalHandler;
+zx_handle_t SignalHandlerEvent = ZX_HANDLE_INVALID;
+
// Helper function to handle Zircon syscall failures.
void ExitOnErr(zx_status_t Status, const char *Syscall) {
if (Status != ZX_OK) {
@@ -68,23 +74,6 @@
}
}
-// CFAOffset is used to reference the stack pointer before entering the
-// trampoline (Stack Pointer + CFAOffset = prev Stack Pointer). Before jumping
-// to the trampoline we copy all the registers onto the stack. We need to make
-// sure that the new stack has enough space to store all the registers.
-//
-// The trampoline holds CFI information regarding the registers stored in the
-// stack, which is then used by the unwinder to restore them.
-#if defined(__x86_64__)
-// In x86_64 the crashing function might also be using the red zone (128 bytes
-// on top of their rsp).
-constexpr size_t CFAOffset = 128 + sizeof(zx_thread_state_general_regs_t);
-#elif defined(__aarch64__)
-// In aarch64 we need to always have the stack pointer aligned to 16 bytes, so we
-// make sure that we are keeping that same alignment.
-constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(uintptr_t)16;
-#endif
-
// For the crash handler, we need to call Fuzzer::StaticCrashSignalCallback
// without POSIX signal handlers. To achieve this, we use an assembly function
// to add the necessary CFI unwinding information and a C function to bridge
@@ -163,10 +152,10 @@
// Produces an assembler immediate operand for the named or numbered register.
// This operand contains the offset of the register relative to the CFA.
-#define ASM_OPERAND_REG(reg) \
- [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg) - CFAOffset),
-#define ASM_OPERAND_NUM(num) \
- [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num]) - CFAOffset),
+#define ASM_OPERAND_REG(reg) \
+ [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg)),
+#define ASM_OPERAND_NUM(num) \
+ [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num])),
// Trampoline to bridge from the assembly below to the static C++ crash
// callback.
@@ -178,62 +167,57 @@
}
}
-// Creates the trampoline with the necessary CFI information to unwind through
-// to the crashing call stack:
-// * Defining the CFA so that it points to the stack pointer at the point
-// of crash.
-// * Storing all registers at the point of crash in the stack and refer to them
-// via CFI information (relative to the CFA).
-// * Setting the return column so the unwinder knows how to continue unwinding.
-// * (x86_64) making sure rsp is aligned before calling StaticCrashHandler.
-// * Calling StaticCrashHandler that will trigger the unwinder.
+// This trampoline function has the necessary CFI information to unwind
+// and get a backtrace:
+// * The stack contains a copy of all the registers at the point of crash,
+// the code has CFI directives specifying how to restore them.
+// * A call to StaticCrashHandler, which will print the stacktrace and exit
+// the fuzzer, generating a crash artifact.
//
// The __attribute__((used)) is necessary because the function
// is never called; it's just a container around the assembly to allow it to
// use operands for compile-time computed constants.
__attribute__((used))
void MakeTrampoline() {
- __asm__(".cfi_endproc\n"
- ".pushsection .text.CrashTrampolineAsm\n"
- ".type CrashTrampolineAsm,STT_FUNC\n"
-"CrashTrampolineAsm:\n"
- ".cfi_startproc simple\n"
- ".cfi_signal_frame\n"
+ __asm__(
+ ".cfi_endproc\n"
+ ".pushsection .text.CrashTrampolineAsm\n"
+ ".type CrashTrampolineAsm,STT_FUNC\n"
+ "CrashTrampolineAsm:\n"
+ ".cfi_startproc simple\n"
+ ".cfi_signal_frame\n"
#if defined(__x86_64__)
- ".cfi_return_column rip\n"
- ".cfi_def_cfa rsp, %c[CFAOffset]\n"
- FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
- "mov %%rsp, %%rbp\n"
- ".cfi_def_cfa_register rbp\n"
- "andq $-16, %%rsp\n"
- "call %c[StaticCrashHandler]\n"
- "ud2\n"
+ ".cfi_return_column rip\n"
+ ".cfi_def_cfa rsp, 0\n"
+ FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
+ "call %c[StaticCrashHandler]\n"
+ "ud2\n"
#elif defined(__aarch64__)
- ".cfi_return_column 33\n"
- ".cfi_def_cfa sp, %c[CFAOffset]\n"
- FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
- ".cfi_offset 33, %c[pc]\n"
- ".cfi_offset 30, %c[lr]\n"
- "bl %c[StaticCrashHandler]\n"
- "brk 1\n"
+ ".cfi_return_column 33\n"
+ ".cfi_def_cfa sp, 0\n"
+ FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM)
+ ".cfi_offset 33, %c[pc]\n"
+ ".cfi_offset 30, %c[lr]\n"
+ "bl %c[StaticCrashHandler]\n"
+ "brk 1\n"
#else
#error "Unsupported architecture for fuzzing on Fuchsia"
#endif
- ".cfi_endproc\n"
- ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n"
- ".popsection\n"
- ".cfi_startproc\n"
- : // No outputs
- : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM)
+ ".cfi_endproc\n"
+ ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n"
+ ".popsection\n"
+ ".cfi_startproc\n"
+ : // No outputs
+ : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM)
#if defined(__aarch64__)
- ASM_OPERAND_REG(pc)
- ASM_OPERAND_REG(lr)
+ ASM_OPERAND_REG(pc) ASM_OPERAND_REG(lr)
#endif
- [StaticCrashHandler] "i" (StaticCrashHandler),
- [CFAOffset] "i" (CFAOffset));
+ [StaticCrashHandler] "i"(StaticCrashHandler));
}
-void CrashHandler(zx_handle_t *Event) {
+void CrashHandler() {
+ assert(SignalHandlerEvent != ZX_HANDLE_INVALID);
+
// This structure is used to ensure we close handles to objects we create in
// this handler.
struct ScopedHandle {
@@ -251,16 +235,30 @@
Self, ZX_EXCEPTION_CHANNEL_DEBUGGER, &Channel.Handle),
"_zx_task_create_exception_channel");
- ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0),
+ ExitOnErr(_zx_object_signal(SignalHandlerEvent, 0, ZX_USER_SIGNAL_0),
"_zx_object_signal");
// This thread lives as long as the process in order to keep handling
// crashes. In practice, the first crashed thread to reach the end of the
// StaticCrashHandler will end the process.
while (true) {
- ExitOnErr(_zx_object_wait_one(Channel.Handle, ZX_CHANNEL_READABLE,
- ZX_TIME_INFINITE, nullptr),
- "_zx_object_wait_one");
+ zx_wait_item_t WaitItems[] = {
+ {
+ .handle = SignalHandlerEvent,
+ .waitfor = ZX_SIGNAL_HANDLE_CLOSED,
+ .pending = 0,
+ },
+ {
+ .handle = Channel.Handle,
+ .waitfor = ZX_CHANNEL_READABLE | ZX_CHANNEL_PEER_CLOSED,
+ .pending = 0,
+ },
+ };
+ auto Status = _zx_object_wait_many(
+ WaitItems, sizeof(WaitItems) / sizeof(WaitItems[0]), ZX_TIME_INFINITE);
+ if (Status != ZX_OK || (WaitItems[1].pending & ZX_CHANNEL_READABLE) == 0) {
+ break;
+ }
zx_exception_info_t ExceptionInfo;
ScopedHandle Exception;
@@ -296,14 +294,17 @@
// onto the stack and jump into a trampoline with CFI instructions on how
// to restore it.
#if defined(__x86_64__)
- uintptr_t StackPtr = GeneralRegisters.rsp - CFAOffset;
+ uintptr_t StackPtr =
+ (GeneralRegisters.rsp - (128 + sizeof(GeneralRegisters))) &
+ -(uintptr_t)16;
__unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters,
sizeof(GeneralRegisters));
GeneralRegisters.rsp = StackPtr;
GeneralRegisters.rip = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm);
#elif defined(__aarch64__)
- uintptr_t StackPtr = GeneralRegisters.sp - CFAOffset;
+ uintptr_t StackPtr =
+ (GeneralRegisters.sp - sizeof(GeneralRegisters)) & -(uintptr_t)16;
__unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters,
sizeof(GeneralRegisters));
GeneralRegisters.sp = StackPtr;
@@ -327,6 +328,13 @@
}
}
+void StopSignalHandler() {
+ _zx_handle_close(SignalHandlerEvent);
+ if (SignalHandler.joinable()) {
+ SignalHandler.join();
+ }
+}
+
} // namespace
// Platform specific functions.
@@ -356,16 +364,14 @@
return;
// Set up the crash handler and wait until it is ready before proceeding.
- zx_handle_t Event;
- ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create");
+ ExitOnErr(_zx_event_create(0, &SignalHandlerEvent), "_zx_event_create");
- std::thread T(CrashHandler, &Event);
- zx_status_t Status =
- _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr);
- _zx_handle_close(Event);
+ SignalHandler = std::thread(CrashHandler);
+ zx_status_t Status = _zx_object_wait_one(SignalHandlerEvent, ZX_USER_SIGNAL_0,
+ ZX_TIME_INFINITE, nullptr);
ExitOnErr(Status, "_zx_object_wait_one");
- T.detach();
+ std::atexit(StopSignalHandler);
}
void SleepSeconds(int Seconds) {
diff --git a/libfuzzer/FuzzerUtilWindows.cpp b/libfuzzer/FuzzerUtilWindows.cpp
index 1a54bb5..3598758 100644
--- a/libfuzzer/FuzzerUtilWindows.cpp
+++ b/libfuzzer/FuzzerUtilWindows.cpp
@@ -204,7 +204,7 @@
}
std::string DisassembleCmd(const std::string &FileName) {
- Vector<std::string> command_vector;
+ std::vector<std::string> command_vector;
command_vector.push_back("dumpbin /summary > nul");
if (ExecuteCommand(Command(command_vector)) == 0)
return "dumpbin /disasm " + FileName;
diff --git a/libfuzzer/dataflow/DataFlow.cpp b/libfuzzer/dataflow/DataFlow.cpp
index 78b3f9a..7e5f041 100644
--- a/libfuzzer/dataflow/DataFlow.cpp
+++ b/libfuzzer/dataflow/DataFlow.cpp
@@ -17,11 +17,9 @@
// and also provides basic-block coverage for every input.
//
// Build:
-// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow -mllvm
-// -dfsan-fast-16-labels and -O2.
+// 1. Compile this file (DataFlow.cpp) with -fsanitize=dataflow and -O2.
// 2. Compile DataFlowCallbacks.cpp with -O2 -fPIC.
// 3. Build the fuzz target with -g -fsanitize=dataflow
-// -mllvm -dfsan-fast-16-labels
// -fsanitize-coverage=trace-pc-guard,pc-table,bb,trace-cmp
// 4. Link those together with -fsanitize=dataflow
//
@@ -82,7 +80,7 @@
return __dft.PCsBeg[BlockIdx * 2 + 1] & PCFLAG_FUNC_ENTRY;
}
-const int kNumLabels = 16;
+const int kNumLabels = 8;
// Prints all instrumented functions.
static int PrintFunctions() {
@@ -92,8 +90,8 @@
// We'll need to make a proper in-process symbolizer work with DFSan.
FILE *Pipe = popen("sed 's/(+/ /g; s/).*//g' "
"| llvm-symbolizer "
- "| grep 'dfs\\$' "
- "| sed 's/dfs\\$//g' "
+ "| grep '\\.dfsan' "
+ "| sed 's/\\.dfsan//g' "
"| c++filt",
"w");
for (size_t I = 0; I < __dft.NumGuards; I++) {
diff --git a/libfuzzer/tests/CMakeLists.txt b/libfuzzer/tests/CMakeLists.txt
index 5b3e906..10fcfba 100644
--- a/libfuzzer/tests/CMakeLists.txt
+++ b/libfuzzer/tests/CMakeLists.txt
@@ -33,7 +33,8 @@
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND
COMPILER_RT_LIBCXX_PATH AND
COMPILER_RT_LIBCXXABI_PATH)
- list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++)
+ list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++ -fno-exceptions)
+ list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -nostdlib++ -fno-exceptions)
endif()
if ("-fvisibility=hidden" IN_LIST LIBFUZZER_CFLAGS)
@@ -73,7 +74,7 @@
FuzzerUnitTests "Fuzzer-${arch}-Test" ${arch}
SOURCES FuzzerUnittest.cpp ${COMPILER_RT_GTEST_SOURCE}
RUNTIME ${LIBFUZZER_TEST_RUNTIME}
- DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS}
+ DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS}
CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS}
LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS})
set_target_properties(FuzzerUnitTests PROPERTIES
@@ -83,7 +84,7 @@
generate_compiler_rt_tests(FuzzedDataProviderTestObjects
FuzzedDataProviderUnitTests "FuzzerUtils-${arch}-Test" ${arch}
SOURCES FuzzedDataProviderUnittest.cpp ${COMPILER_RT_GTEST_SOURCE}
- DEPS gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h
+ DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h
CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS}
LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS})
set_target_properties(FuzzedDataProviderUnitTests PROPERTIES
diff --git a/libfuzzer/tests/FuzzerUnittest.cpp b/libfuzzer/tests/FuzzerUnittest.cpp
index 974a01f..5315a8e 100644
--- a/libfuzzer/tests/FuzzerUnittest.cpp
+++ b/libfuzzer/tests/FuzzerUnittest.cpp
@@ -24,7 +24,8 @@
using namespace fuzzer;
// For now, have LLVMFuzzerTestOneInput just to make it link.
-// Later we may want to make unittests that actually call LLVMFuzzerTestOneInput.
+// Later we may want to make unittests that actually call
+// LLVMFuzzerTestOneInput.
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
abort();
}
@@ -88,7 +89,7 @@
{ 0, 5, 6, 7, 1, 2 }
};
for (size_t Len = 1; Len < 8; Len++) {
- Set<Unit> FoundUnits, ExpectedUnitsWitThisLength;
+ std::set<Unit> FoundUnits, ExpectedUnitsWitThisLength;
for (int Iter = 0; Iter < 3000; Iter++) {
C.resize(Len);
size_t NewSize = MD->CrossOver(A.data(), A.size(), B.data(), B.size(),
@@ -242,7 +243,8 @@
}
TEST(FuzzerMutate, InsertRepeatedBytes1) {
- TestInsertRepeatedBytes(&MutationDispatcher::Mutate_InsertRepeatedBytes, 10000);
+ TestInsertRepeatedBytes(&MutationDispatcher::Mutate_InsertRepeatedBytes,
+ 10000);
}
TEST(FuzzerMutate, InsertRepeatedBytes2) {
TestInsertRepeatedBytes(&MutationDispatcher::Mutate, 300000);
@@ -557,7 +559,7 @@
}
TEST(FuzzerDictionary, ParseDictionaryFile) {
- Vector<Unit> Units;
+ std::vector<Unit> Units;
EXPECT_FALSE(ParseDictionaryFile("zzz\n", &Units));
EXPECT_FALSE(ParseDictionaryFile("", &Units));
EXPECT_TRUE(ParseDictionaryFile("\n", &Units));
@@ -569,11 +571,11 @@
EXPECT_TRUE(ParseDictionaryFile(" #zzzz\n", &Units));
EXPECT_EQ(Units.size(), 0U);
EXPECT_TRUE(ParseDictionaryFile(" #zzzz\naaa=\"aa\"", &Units));
- EXPECT_EQ(Units, Vector<Unit>({Unit({'a', 'a'})}));
+ EXPECT_EQ(Units, std::vector<Unit>({Unit({'a', 'a'})}));
EXPECT_TRUE(
ParseDictionaryFile(" #zzzz\naaa=\"aa\"\n\nabc=\"abc\"", &Units));
EXPECT_EQ(Units,
- Vector<Unit>({Unit({'a', 'a'}), Unit({'a', 'b', 'c'})}));
+ std::vector<Unit>({Unit({'a', 'a'}), Unit({'a', 'b', 'c'})}));
}
TEST(FuzzerUtil, Base64) {
@@ -589,6 +591,42 @@
EXPECT_EQ("YWJjeHl6", Base64({'a', 'b', 'c', 'x', 'y', 'z'}));
}
+#ifdef __GLIBC__
+class PrintfCapture {
+ public:
+ PrintfCapture() {
+ OldOutputFile = GetOutputFile();
+ SetOutputFile(open_memstream(&Buffer, &Size));
+ }
+ ~PrintfCapture() {
+ fclose(GetOutputFile());
+ SetOutputFile(OldOutputFile);
+ free(Buffer);
+ }
+ std::string str() { return std::string(Buffer, Size); }
+
+ private:
+ char *Buffer;
+ size_t Size;
+ FILE *OldOutputFile;
+};
+
+TEST(FuzzerUtil, PrintASCII) {
+ auto f = [](const char *Str, const char *PrintAfter = "") {
+ PrintfCapture Capture;
+ PrintASCII(reinterpret_cast<const uint8_t*>(Str), strlen(Str), PrintAfter);
+ return Capture.str();
+ };
+ EXPECT_EQ("hello", f("hello"));
+ EXPECT_EQ("c:\\\\", f("c:\\"));
+ EXPECT_EQ("\\\"hi\\\"", f("\"hi\""));
+ EXPECT_EQ("\\011a", f("\ta"));
+ EXPECT_EQ("\\0111", f("\t1"));
+ EXPECT_EQ("hello\\012", f("hello\n"));
+ EXPECT_EQ("hello\n", f("hello", "\n"));
+}
+#endif
+
TEST(Corpus, Distribution) {
DataFlowTrace DFT;
Random Rand(0);
@@ -604,7 +642,7 @@
/*FeatureSet*/ {}, DFT,
/*BaseII*/ nullptr);
- Vector<size_t> Hist(N);
+ std::vector<size_t> Hist(N);
for (size_t i = 0; i < N * TriesPerUnit; i++) {
Hist[C->ChooseUnitIdxToMutate(Rand)]++;
}
@@ -614,19 +652,60 @@
}
}
-template <typename T> void EQ(const Vector<T> &A, const Vector<T> &B) {
+TEST(Corpus, Replace) {
+ DataFlowTrace DFT;
+ struct EntropicOptions Entropic = {false, 0xFF, 100, false};
+ std::unique_ptr<InputCorpus> C(
+ new InputCorpus(/*OutputCorpus*/ "", Entropic));
+ InputInfo *FirstII =
+ C->AddToCorpus(Unit{0x01, 0x00}, /*NumFeatures*/ 1,
+ /*MayDeleteFile*/ false, /*HasFocusFunction*/ false,
+ /*ForceAddToCorpus*/ false,
+ /*TimeOfUnit*/ std::chrono::microseconds(1234),
+ /*FeatureSet*/ {}, DFT,
+ /*BaseII*/ nullptr);
+ InputInfo *SecondII =
+ C->AddToCorpus(Unit{0x02}, /*NumFeatures*/ 1,
+ /*MayDeleteFile*/ false, /*HasFocusFunction*/ false,
+ /*ForceAddToCorpus*/ false,
+ /*TimeOfUnit*/ std::chrono::microseconds(5678),
+ /*FeatureSet*/ {}, DFT,
+ /*BaseII*/ nullptr);
+ Unit ReplacedU = Unit{0x03};
+
+ C->Replace(FirstII, ReplacedU,
+ /*TimeOfUnit*/ std::chrono::microseconds(321));
+
+ // FirstII should be replaced.
+ EXPECT_EQ(FirstII->U, Unit{0x03});
+ EXPECT_EQ(FirstII->Reduced, true);
+ EXPECT_EQ(FirstII->TimeOfUnit, std::chrono::microseconds(321));
+ std::vector<uint8_t> ExpectedSha1(kSHA1NumBytes);
+ ComputeSHA1(ReplacedU.data(), ReplacedU.size(), ExpectedSha1.data());
+ std::vector<uint8_t> IISha1(FirstII->Sha1, FirstII->Sha1 + kSHA1NumBytes);
+ EXPECT_EQ(IISha1, ExpectedSha1);
+
+ // SecondII should not be replaced.
+ EXPECT_EQ(SecondII->U, Unit{0x02});
+ EXPECT_EQ(SecondII->Reduced, false);
+ EXPECT_EQ(SecondII->TimeOfUnit, std::chrono::microseconds(5678));
+}
+
+template <typename T>
+void EQ(const std::vector<T> &A, const std::vector<T> &B) {
EXPECT_EQ(A, B);
}
-template <typename T> void EQ(const Set<T> &A, const Vector<T> &B) {
- EXPECT_EQ(A, Set<T>(B.begin(), B.end()));
+template <typename T> void EQ(const std::set<T> &A, const std::vector<T> &B) {
+ EXPECT_EQ(A, std::set<T>(B.begin(), B.end()));
}
-void EQ(const Vector<MergeFileInfo> &A, const Vector<std::string> &B) {
- Set<std::string> a;
+void EQ(const std::vector<MergeFileInfo> &A,
+ const std::vector<std::string> &B) {
+ std::set<std::string> a;
for (const auto &File : A)
a.insert(File.Name);
- Set<std::string> b(B.begin(), B.end());
+ std::set<std::string> b(B.begin(), B.end());
EXPECT_EQ(a, b);
}
@@ -746,9 +825,9 @@
TEST(Merger, Merge) {
Merger M;
- Set<uint32_t> Features, NewFeatures;
- Set<uint32_t> Cov, NewCov;
- Vector<std::string> NewFiles;
+ std::set<uint32_t> Features, NewFeatures;
+ std::set<uint32_t> Cov, NewCov;
+ std::vector<std::string> NewFiles;
// Adds new files and features
EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n"
@@ -861,6 +940,137 @@
TRACED_EQ(NewFeatures, {1, 2, 3});
}
+TEST(Merger, SetCoverMerge) {
+ Merger M;
+ std::set<uint32_t> Features, NewFeatures;
+ std::set<uint32_t> Cov, NewCov;
+ std::vector<std::string> NewFiles;
+
+ // Adds new files and features
+ EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 6U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {"A", "B"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6});
+
+ // Doesn't return features or files in the initial corpus.
+ EXPECT_TRUE(M.Parse("3\n1\nA\nB\nC\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 3U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {"B"});
+ TRACED_EQ(NewFeatures, {4, 5, 6});
+
+ // No new features, so no new files
+ EXPECT_TRUE(M.Parse("3\n2\nA\nB\nC\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1002\n"
+ "FT 2 6 1 3\n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 0U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ TRACED_EQ(NewFiles, {});
+ TRACED_EQ(NewFeatures, {});
+
+ // Can pass initial features and coverage.
+ Features = {1, 2, 3};
+ Cov = {};
+ EXPECT_TRUE(M.Parse("2\n0\nA\nB\n"
+ "STARTED 0 1000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1001\n"
+ "FT 1 4 5 6\n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 3U);
+ TRACED_EQ(M.Files, {"A", "B"});
+ TRACED_EQ(NewFiles, {"B"});
+ TRACED_EQ(NewFeatures, {4, 5, 6});
+ Features.clear();
+ Cov.clear();
+
+ // Prefer files with a lot of features first (C has 4 features)
+ // Then prefer B over A due to the smaller size. After choosing C and B,
+ // A and D have no new features to contribute.
+ EXPECT_TRUE(M.Parse("4\n0\nA\nB\nC\nD\n"
+ "STARTED 0 2000\n"
+ "FT 0 3 5 6\n"
+ "STARTED 1 1000\n"
+ "FT 1 4 5 6 \n"
+ "STARTED 2 1000\n"
+ "FT 2 1 2 3 4 \n"
+ "STARTED 3 500\n"
+ "FT 3 1 \n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 6U);
+ TRACED_EQ(M.Files, {"A", "B", "C", "D"});
+ TRACED_EQ(NewFiles, {"C", "B"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5, 6});
+
+ // Only 1 file covers all features.
+ EXPECT_TRUE(M.Parse("4\n1\nA\nB\nC\nD\n"
+ "STARTED 0 2000\n"
+ "FT 0 4 5 6 7 8\n"
+ "STARTED 1 1100\n"
+ "FT 1 1 2 3 \n"
+ "STARTED 2 1100\n"
+ "FT 2 2 3 \n"
+ "STARTED 3 1000\n"
+ "FT 3 1 \n"
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 3U);
+ TRACED_EQ(M.Files, {"A", "B", "C", "D"});
+ TRACED_EQ(NewFiles, {"B"});
+ TRACED_EQ(NewFeatures, {1, 2, 3});
+
+ // A Feature has a value greater than (1 << 21) and hence
+ // there are collisions in the underlying `covered features`
+ // bitvector.
+ EXPECT_TRUE(M.Parse("3\n0\nA\nB\nC\n"
+ "STARTED 0 2000\n"
+ "FT 0 1 2 3\n"
+ "STARTED 1 1000\n"
+ "FT 1 3 4 5 \n"
+ "STARTED 2 1000\n"
+ "FT 2 3 2097153 \n" // Last feature is (2^21 + 1).
+ "",
+ true));
+ EXPECT_EQ(M.SetCoverMerge(Features, &NewFeatures, Cov, &NewCov, &NewFiles),
+ 5U);
+ TRACED_EQ(M.Files, {"A", "B", "C"});
+ // File 'C' is not added because it's last feature is considered
+ // covered due to collision with feature 1.
+ TRACED_EQ(NewFiles, {"B", "A"});
+ TRACED_EQ(NewFeatures, {1, 2, 3, 4, 5});
+}
+
#undef TRACED_EQ
TEST(DFT, BlockCoverage) {
@@ -968,7 +1178,7 @@
0, 0, 0, 0, 0, 0, 0, 8,
9, 9, 9, 9, 9, 9, 9, 9,
};
- typedef Vector<std::pair<size_t, uint8_t> > Vec;
+ typedef std::vector<std::pair<size_t, uint8_t>> Vec;
Vec Res, Expected;
auto CB = [&](size_t FirstFeature, size_t Idx, uint8_t V) {
Res.push_back({FirstFeature + Idx, V});
@@ -993,7 +1203,7 @@
// FuzzerCommand unit tests. The arguments in the two helper methods below must
// match.
-static void makeCommandArgs(Vector<std::string> *ArgsToAdd) {
+static void makeCommandArgs(std::vector<std::string> *ArgsToAdd) {
assert(ArgsToAdd);
ArgsToAdd->clear();
ArgsToAdd->push_back("foo");
@@ -1029,7 +1239,7 @@
EXPECT_EQ(CmdLine, "");
// Explicit constructor
- Vector<std::string> ArgsToAdd;
+ std::vector<std::string> ArgsToAdd;
makeCommandArgs(&ArgsToAdd);
Command InitializedCmd(ArgsToAdd);
@@ -1061,7 +1271,7 @@
}
TEST(FuzzerCommand, ModifyArguments) {
- Vector<std::string> ArgsToAdd;
+ std::vector<std::string> ArgsToAdd;
makeCommandArgs(&ArgsToAdd);
Command Cmd;
std::string CmdLine;
@@ -1084,7 +1294,7 @@
}
TEST(FuzzerCommand, ModifyFlags) {
- Vector<std::string> ArgsToAdd;
+ std::vector<std::string> ArgsToAdd;
makeCommandArgs(&ArgsToAdd);
Command Cmd(ArgsToAdd);
std::string Value, CmdLine;
@@ -1116,7 +1326,7 @@
}
TEST(FuzzerCommand, SetOutput) {
- Vector<std::string> ArgsToAdd;
+ std::vector<std::string> ArgsToAdd;
makeCommandArgs(&ArgsToAdd);
Command Cmd(ArgsToAdd);
std::string CmdLine;
@@ -1196,7 +1406,8 @@
struct EntropicOptions Entropic = {true, 0xFF, 100, false};
std::unique_ptr<InputCorpus> C(new InputCorpus("", Entropic));
std::unique_ptr<InputInfo> II(new InputInfo());
- Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs = {{1, 3}, {2, 3}, {3, 3}};
+ std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs = {
+ {1, 3}, {2, 3}, {3, 3}};
II->FeatureFreqs = FeatureFreqs;
II->NumExecutedMutations = 0;
II->UpdateEnergy(4, false, std::chrono::microseconds(0));
diff --git a/src/lib.rs b/src/lib.rs
index 6badc39..ce1eb98 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -14,10 +14,41 @@
pub use arbitrary;
use once_cell::sync::OnceCell;
+/// Indicates whether the input should be kept in the corpus or rejected. This
+/// should be returned by your fuzz target. If your fuzz target does not return
+/// a value (i.e., returns `()`), then the input will be kept in the corpus.
+#[derive(Debug)]
+pub enum Corpus {
+ /// Keep the input in the corpus.
+ Keep,
+
+ /// Reject the input and do not keep it in the corpus.
+ Reject,
+}
+
+impl From<()> for Corpus {
+ fn from(_: ()) -> Self {
+ Self::Keep
+ }
+}
+
+impl Corpus {
+ #[doc(hidden)]
+ /// Convert this Corpus result into the [integer codes used by
+ /// `libFuzzer`](https://llvm.org/docs/LibFuzzer.html#rejecting-unwanted-inputs).
+ /// This is -1 for reject, 0 for keep.
+ pub fn to_libfuzzer_code(self) -> i32 {
+ match self {
+ Corpus::Keep => 0,
+ Corpus::Reject => -1,
+ }
+ }
+}
+
extern "C" {
// We do not actually cross the FFI bound here.
#[allow(improper_ctypes)]
- fn rust_fuzzer_test_input(input: &[u8]);
+ fn rust_fuzzer_test_input(input: &[u8]) -> i32;
fn LLVMFuzzerMutate(data: *mut u8, size: usize, max_size: usize) -> usize;
}
@@ -27,14 +58,17 @@
pub fn test_input_wrap(data: *const u8, size: usize) -> i32 {
let test_input = ::std::panic::catch_unwind(|| unsafe {
let data_slice = ::std::slice::from_raw_parts(data, size);
- rust_fuzzer_test_input(data_slice);
+ rust_fuzzer_test_input(data_slice)
});
- if test_input.err().is_some() {
- // hopefully the custom panic hook will be called before and abort the
- // process before the stack frames are unwinded.
- ::std::process::abort();
+
+ match test_input {
+ Ok(i) => i,
+ Err(_) => {
+ // hopefully the custom panic hook will be called before and abort the
+ // process before the stack frames are unwinded.
+ ::std::process::abort();
+ }
}
- 0
}
#[doc(hidden)]
@@ -86,6 +120,39 @@
/// # mod my_crate { pub fn parse(_: &[u8]) -> Result<(), ()> { unimplemented!() } }
/// ```
///
+/// ## Rejecting Inputs
+///
+/// It may be desirable to reject some inputs, i.e. to not add them to the
+/// corpus.
+///
+/// For example, when fuzzing an API consisting of parsing and other logic,
+/// one may want to allow only those inputs into the corpus that parse
+/// successfully. To indicate whether an input should be kept in or rejected
+/// from the corpus, return either [Corpus::Keep] or [Corpus::Reject] from your
+/// fuzz target. The default behavior (e.g. if `()` is returned) is to keep the
+/// input in the corpus.
+///
+/// For example:
+///
+/// ```no_run
+/// #![no_main]
+///
+/// use libfuzzer_sys::{Corpus, fuzz_target};
+///
+/// fuzz_target!(|input: String| -> Corpus {
+/// let parts: Vec<&str> = input.splitn(2, '=').collect();
+/// if parts.len() != 2 {
+/// return Corpus::Reject;
+/// }
+///
+/// let key = parts[0];
+/// let value = parts[1];
+/// let _result: Result<_, _> = my_crate::parse(key, value);
+/// Corpus::Keep
+/// });
+/// # mod my_crate { pub fn parse(_key: &str, _value: &str) -> Result<(), ()> { unimplemented!() } }
+/// ```
+///
/// ## Arbitrary Input Types
///
/// The input is a `&[u8]` slice by default, but you can take arbitrary input
@@ -136,74 +203,107 @@
#[macro_export]
macro_rules! fuzz_target {
(|$bytes:ident| $body:block) => {
- /// Auto-generated function
- #[no_mangle]
- pub extern "C" fn rust_fuzzer_test_input($bytes: &[u8]) {
- // When `RUST_LIBFUZZER_DEBUG_PATH` is set, write the debug
- // formatting of the input to that file. This is only intended for
- // `cargo fuzz`'s use!
+ const _: () = {
+ /// Auto-generated function
+ #[no_mangle]
+ pub extern "C" fn rust_fuzzer_test_input(bytes: &[u8]) -> i32 {
+ // When `RUST_LIBFUZZER_DEBUG_PATH` is set, write the debug
+ // formatting of the input to that file. This is only intended for
+ // `cargo fuzz`'s use!
- // `RUST_LIBFUZZER_DEBUG_PATH` is set in initialization.
- if let Some(path) = $crate::RUST_LIBFUZZER_DEBUG_PATH.get() {
- use std::io::Write;
- let mut file = std::fs::File::create(path)
- .expect("failed to create `RUST_LIBFUZZER_DEBUG_PATH` file");
- writeln!(&mut file, "{:?}", $bytes)
- .expect("failed to write to `RUST_LIBFUZZER_DEBUG_PATH` file");
- return;
+ // `RUST_LIBFUZZER_DEBUG_PATH` is set in initialization.
+ if let Some(path) = $crate::RUST_LIBFUZZER_DEBUG_PATH.get() {
+ use std::io::Write;
+ let mut file = std::fs::File::create(path)
+ .expect("failed to create `RUST_LIBFUZZER_DEBUG_PATH` file");
+ writeln!(&mut file, "{:?}", bytes)
+ .expect("failed to write to `RUST_LIBFUZZER_DEBUG_PATH` file");
+ return 0;
+ }
+
+ run(bytes);
+ 0
}
- $body
- }
+ // Split out the actual fuzzer into a separate function which is
+ // tagged as never being inlined. This ensures that if the fuzzer
+ // panics there's at least one stack frame which is named uniquely
+ // according to this specific fuzzer that this is embedded within.
+ //
+ // Systems like oss-fuzz try to deduplicate crashes and without this
+ // panics in separate fuzzers can accidentally appear the same
+ // because each fuzzer will have a function called
+ // `rust_fuzzer_test_input`. By using a normal Rust function here
+ // it's named something like `the_fuzzer_name::_::run` which should
+ // ideally help prevent oss-fuzz from deduplicate fuzz bugs across
+ // distinct targets accidentally.
+ #[inline(never)]
+ fn run($bytes: &[u8]) {
+ $body
+ }
+ };
};
(|$data:ident: &[u8]| $body:block) => {
- fuzz_target!(|$data| $body);
+ $crate::fuzz_target!(|$data| $body);
};
(|$data:ident: $dty: ty| $body:block) => {
- /// Auto-generated function
- #[no_mangle]
- pub extern "C" fn rust_fuzzer_test_input(bytes: &[u8]) {
- use $crate::arbitrary::{Arbitrary, Unstructured};
+ $crate::fuzz_target!(|$data: $dty| -> () $body);
+ };
- // Early exit if we don't have enough bytes for the `Arbitrary`
- // implementation. This helps the fuzzer avoid exploring all the
- // different not-enough-input-bytes paths inside the `Arbitrary`
- // implementation. Additionally, it exits faster, letting the fuzzer
- // get to longer inputs that actually lead to interesting executions
- // quicker.
- if bytes.len() < <$dty as Arbitrary>::size_hint(0).0 {
- return;
+ (|$data:ident: $dty: ty| -> $rty: ty $body:block) => {
+ const _: () = {
+ /// Auto-generated function
+ #[no_mangle]
+ pub extern "C" fn rust_fuzzer_test_input(bytes: &[u8]) -> i32 {
+ use $crate::arbitrary::{Arbitrary, Unstructured};
+
+ // Early exit if we don't have enough bytes for the `Arbitrary`
+ // implementation. This helps the fuzzer avoid exploring all the
+ // different not-enough-input-bytes paths inside the `Arbitrary`
+ // implementation. Additionally, it exits faster, letting the fuzzer
+ // get to longer inputs that actually lead to interesting executions
+ // quicker.
+ if bytes.len() < <$dty as Arbitrary>::size_hint(0).0 {
+ return -1;
+ }
+
+ let mut u = Unstructured::new(bytes);
+ let data = <$dty as Arbitrary>::arbitrary_take_rest(u);
+
+ // When `RUST_LIBFUZZER_DEBUG_PATH` is set, write the debug
+ // formatting of the input to that file. This is only intended for
+ // `cargo fuzz`'s use!
+
+ // `RUST_LIBFUZZER_DEBUG_PATH` is set in initialization.
+ if let Some(path) = $crate::RUST_LIBFUZZER_DEBUG_PATH.get() {
+ use std::io::Write;
+ let mut file = std::fs::File::create(path)
+ .expect("failed to create `RUST_LIBFUZZER_DEBUG_PATH` file");
+ (match data {
+ Ok(data) => writeln!(&mut file, "{:#?}", data),
+ Err(err) => writeln!(&mut file, "Arbitrary Error: {}", err),
+ })
+ .expect("failed to write to `RUST_LIBFUZZER_DEBUG_PATH` file");
+ return -1;
+ }
+
+ let data = match data {
+ Ok(d) => d,
+ Err(_) => return -1,
+ };
+
+ let result = ::libfuzzer_sys::Corpus::from(run(data));
+ result.to_libfuzzer_code()
}
- let mut u = Unstructured::new(bytes);
- let data = <$dty as Arbitrary>::arbitrary_take_rest(u);
-
- // When `RUST_LIBFUZZER_DEBUG_PATH` is set, write the debug
- // formatting of the input to that file. This is only intended for
- // `cargo fuzz`'s use!
-
- // `RUST_LIBFUZZER_DEBUG_PATH` is set in initialization.
- if let Some(path) = $crate::RUST_LIBFUZZER_DEBUG_PATH.get() {
- use std::io::Write;
- let mut file = std::fs::File::create(path)
- .expect("failed to create `RUST_LIBFUZZER_DEBUG_PATH` file");
- (match data {
- Ok(data) => writeln!(&mut file, "{:#?}", data),
- Err(err) => writeln!(&mut file, "Arbitrary Error: {}", err),
- })
- .expect("failed to write to `RUST_LIBFUZZER_DEBUG_PATH` file");
- return;
+ // See above for why this is split to a separate function.
+ #[inline(never)]
+ fn run($data: $dty) -> $rty {
+ $body
}
-
- let $data = match data {
- Ok(d) => d,
- Err(_) => return,
- };
-
- $body
- }
+ };
};
}
