Test standard and persistent modes separately
diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index cf9976f..bbc166e 100644
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -3,9 +3,9 @@
# Author: Chris Ball <chris@printf.net>
# Ported from Marc "van Hauser" Heuse's "benchmark.sh".
import os
-import subprocess
-import shutil
import re
+import shutil
+import subprocess
import sys
def colon_value_or_none(filename: str, searchKey: str) -> str | None:
@@ -20,6 +20,16 @@
return value
return None
+def compile_target(source: str, binary: str) -> None:
+ with open("afl.log", "w") as f:
+ process = subprocess.run(
+ ["afl-cc", "-o", binary, source],
+ stdout=f,
+ stderr=subprocess.STDOUT,
+ env={"AFL_INSTRUMENT": "PCGUARD", "PATH": os.environ["PATH"]}
+ )
+ if process.returncode != 0:
+ sys.exit("Error: afl-cc is unable to compile")
# Check if the necessary files exist and are executable
if not (
@@ -27,13 +37,15 @@
and os.access("../afl-cc", os.X_OK)
and os.path.exists("../SanitizerCoveragePCGUARD.so")
):
- print(
- "Error: you need to compile AFL++ first, we need afl-fuzz, afl-clang-fast and SanitizerCoveragePCGUARD.so built."
- )
- exit(1)
+ sys.exit("Error: you need to compile AFL++ first, we need afl-fuzz, afl-clang-fast and SanitizerCoveragePCGUARD.so built.")
print("Preparing environment")
+targets = [
+ {"source": "../test-instr.c", "binary": "test-instr"},
+ {"source": "../utils/persistent_mode/test-instr.c", "binary": "test-instr-persistent"}
+]
+
# Unset AFL_* environment variables
for e in list(os.environ.keys()):
if e.startswith("AFL_"):
@@ -42,17 +54,8 @@
AFL_PATH = os.path.abspath("../")
os.environ["PATH"] = AFL_PATH + ":" + os.environ["PATH"]
-# Compile test-instr.c
-with open("afl.log", "w") as f:
- process = subprocess.run(
- ["../afl-cc", "-o", "test-instr", "../test-instr.c"],
- stdout=f,
- stderr=subprocess.STDOUT,
- env={"AFL_INSTRUMENT": "PCGUARD"}
- )
- if process.returncode != 0:
- print("Error: afl-cc is unable to compile")
- exit(1)
+for target in targets:
+ compile_target(target["source"], target["binary"])
# Create input directory and file
os.makedirs("in", exist_ok=True)
@@ -69,33 +72,34 @@
"AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES": "1",
"AFL_BENCH_JUST_ONE": "1",
}
-with open("afl.log", "a") as f:
- process = subprocess.run(
- [
- "afl-fuzz",
- "-i",
- "in",
- "-o",
- "out",
- "-s",
- "123",
- "-D",
- "./test-instr",
- ],
- stdout=f,
- stderr=subprocess.STDOUT,
- env={**os.environ, **env_vars},
- )
+
+for target in targets:
+ with open(f"afl-{target['binary']}.log", "a") as f:
+ process = subprocess.run(
+ [
+ "afl-fuzz",
+ "-i",
+ "in",
+ "-o",
+ f"out-{target['binary']}",
+ "-s",
+ "123",
+ "-D",
+ f"./{target['binary']}",
+ ],
+ stdout=f,
+ stderr=subprocess.STDOUT,
+ env={**os.environ, **env_vars},
+ )
print("Analysis:")
# Extract CPUID from afl.log
-with open("afl.log", "r") as f:
+with open(f"afl-test-instr.log", "r") as f:
match = re.search(r".*try binding to.*#(\d+)", f.read())
if not match:
sys.exit("Couldn't see which CPU# was used in afl.log", 1)
cpuid = match.group(1)
- print(cpuid)
# Print CPU model
model = colon_value_or_none("/proc/cpuinfo", "model name")
@@ -120,16 +124,19 @@
print(" Mhz:", cpu_speed)
# Print execs_per_sec from fuzzer_stats
-execs = colon_value_or_none("out/default/fuzzer_stats", "execs_per_sec")
-if execs:
- print(" execs/s:", execs)
+for target in targets:
+ execs = colon_value_or_none(f"out-{target['binary']}/default/fuzzer_stats", "execs_per_sec")
+ if execs:
+ print(f" {target['binary']} single-core execs/s:", execs)
print("\nComparison: (note that values can change by 10-15% per run)")
with open("COMPARISON", "r") as f:
print(f.read())
# Clean up
-shutil.rmtree("in")
-shutil.rmtree("out")
-os.remove("test-instr")
os.remove("afl.log")
+shutil.rmtree("in")
+for target in targets:
+ shutil.rmtree(f"out-{target['binary']}")
+ os.remove(target["binary"])
+
