junit: Use experimental mutator in regression tests
diff --git a/examples/junit/src/test/java/com/example/MutatorFuzzTest.java b/examples/junit/src/test/java/com/example/MutatorFuzzTest.java
index 2c293b1..f364479 100644
--- a/examples/junit/src/test/java/com/example/MutatorFuzzTest.java
+++ b/examples/junit/src/test/java/com/example/MutatorFuzzTest.java
@@ -36,7 +36,11 @@
@AfterAll
static void assertFuzzTargetRunner() {
- assertTrue(FuzzTargetRunner.invalidCorpusFilesPresent());
- assertEquals(FuzzTargetRunner.mutatorDebugString(), "Arguments[Nullable<List<String>>]");
+ // FuzzTargetRunner values are not set in JUnit engine tests.
+ String jazzerFuzz = System.getenv("JAZZER_FUZZ");
+ if (jazzerFuzz != null && !jazzerFuzz.isEmpty()) {
+ assertTrue(FuzzTargetRunner.invalidCorpusFilesPresent());
+ assertEquals(FuzzTargetRunner.mutatorDebugString(), "Arguments[Nullable<List<String>>]");
+ }
}
}
diff --git a/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetRunner.java b/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetRunner.java
index bae4396..17d5173 100644
--- a/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetRunner.java
+++ b/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetRunner.java
@@ -116,9 +116,9 @@
if (Opt.experimentalMutator) {
if (Modifier.isStatic(fuzzTarget.method.getModifiers())) {
- mutator = ArgumentsMutator.forStaticMethod(fuzzTarget.method);
+ mutator = ArgumentsMutator.forStaticMethodOrThrow(fuzzTarget.method);
} else {
- mutator = ArgumentsMutator.forInstanceMethod(fuzzTargetInstance, fuzzTarget.method);
+ mutator = ArgumentsMutator.forInstanceMethodOrThrow(fuzzTargetInstance, fuzzTarget.method);
}
Log.info("Using experimental mutator: " + mutator);
} else {
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel
index 49792fd..823490b 100644
--- a/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel
+++ b/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel
@@ -44,6 +44,7 @@
"//src/main/java/com/code_intelligence/jazzer/autofuzz",
"//src/main/java/com/code_intelligence/jazzer/driver:fuzzed_data_provider_impl",
"//src/main/java/com/code_intelligence/jazzer/driver:opt",
+ "//src/main/java/com/code_intelligence/jazzer/mutation",
"@maven//:org_junit_jupiter_junit_jupiter_api",
"@maven//:org_junit_jupiter_junit_jupiter_params",
],
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestArgumentsProvider.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestArgumentsProvider.java
index 9f2ba00..90d7f48 100644
--- a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestArgumentsProvider.java
+++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestArgumentsProvider.java
@@ -22,6 +22,8 @@
import com.code_intelligence.jazzer.autofuzz.Meta;
import com.code_intelligence.jazzer.driver.FuzzedDataProviderImpl;
import com.code_intelligence.jazzer.driver.Opt;
+import com.code_intelligence.jazzer.mutation.ArgumentsMutator;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URI;
@@ -34,8 +36,10 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.AbstractMap.SimpleEntry;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Stream;
import org.junit.jupiter.api.extension.ExtensionContext;
@@ -116,20 +120,30 @@
return rawSeeds.map(
e -> arguments(named(e.getKey(), FuzzedDataProviderImpl.withJavaData(e.getValue()))));
} else {
- // Use Autofuzz on the @FuzzTest method.
+ // Use Autofuzz or mutation framework on the @FuzzTest method.
+ Optional<ArgumentsMutator> argumentsMutator =
+ Opt.experimentalMutator ? ArgumentsMutator.forMethod(fuzzTestMethod) : Optional.empty();
+
return rawSeeds.map(e -> {
- try (FuzzedDataProviderImpl data = FuzzedDataProviderImpl.withJavaData(e.getValue())) {
- // The Autofuzz FuzzTarget uses data to construct an instance of the test class before it
- // constructs the fuzz test arguments. We don't need the instance here, but still generate
- // it as that mutates the FuzzedDataProvider state.
- Meta meta = new Meta(fuzzTestMethod.getDeclaringClass());
- meta.consumeNonStatic(data, fuzzTestMethod.getDeclaringClass());
- Object[] args = meta.consumeArguments(data, fuzzTestMethod, null);
- // In order to name the subtest, we name the first argument. All other arguments are
- // passed in unchanged.
- args[0] = named(e.getKey(), args[0]);
- return arguments(args);
+ Object[] args;
+ if (argumentsMutator.isPresent()) {
+ ArgumentsMutator mutator = argumentsMutator.get();
+ mutator.read(new ByteArrayInputStream(e.getValue()));
+ args = mutator.getArguments();
+ } else {
+ try (FuzzedDataProviderImpl data = FuzzedDataProviderImpl.withJavaData(e.getValue())) {
+ // The Autofuzz FuzzTarget uses data to construct an instance of the test class before
+ // it constructs the fuzz test arguments. We don't need the instance here, but still
+ // generate it as that mutates the FuzzedDataProvider state.
+ Meta meta = new Meta(fuzzTestMethod.getDeclaringClass());
+ meta.consumeNonStatic(data, fuzzTestMethod.getDeclaringClass());
+ args = meta.consumeArguments(data, fuzzTestMethod, null);
+ }
}
+ // In order to name the subtest, we name the first argument. All other arguments are
+ // passed in unchanged.
+ args[0] = named(e.getKey(), args[0]);
+ return arguments(args);
});
}
}
diff --git a/src/main/java/com/code_intelligence/jazzer/mutation/ArgumentsMutator.java b/src/main/java/com/code_intelligence/jazzer/mutation/ArgumentsMutator.java
index 1347118..4e5cf9b 100644
--- a/src/main/java/com/code_intelligence/jazzer/mutation/ArgumentsMutator.java
+++ b/src/main/java/com/code_intelligence/jazzer/mutation/ArgumentsMutator.java
@@ -45,6 +45,7 @@
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.util.Arrays;
import java.util.Optional;
public final class ArgumentsMutator {
@@ -71,20 +72,39 @@
stream(method.getAnnotatedParameterTypes()).map(Object::toString).collect(joining(", ")));
}
- public static ArgumentsMutator forInstanceMethod(Object instance, Method method) {
+ public static ArgumentsMutator forMethodOrThrow(Method method) {
+ return forMethod(Mutators.newFactory(), null, method)
+ .orElseThrow(()
+ -> new IllegalArgumentException(
+ "Failed to construct mutator for " + prettyPrintMethod(method)));
+ }
+
+ public static ArgumentsMutator forInstanceMethodOrThrow(Object instance, Method method) {
return forInstanceMethod(Mutators.newFactory(), instance, method)
.orElseThrow(()
-> new IllegalArgumentException(
"Failed to construct mutator for " + prettyPrintMethod(method)));
}
- public static ArgumentsMutator forStaticMethod(Method method) {
+ public static ArgumentsMutator forStaticMethodOrThrow(Method method) {
return forStaticMethod(Mutators.newFactory(), method)
.orElseThrow(()
-> new IllegalArgumentException(
"Failed to construct mutator for " + prettyPrintMethod(method)));
}
+ public static Optional<ArgumentsMutator> forMethod(Method method) {
+ return forMethod(Mutators.newFactory(), null, method);
+ }
+
+ public static Optional<ArgumentsMutator> forInstanceMethod(Object instance, Method method) {
+ return forInstanceMethod(Mutators.newFactory(), instance, method);
+ }
+
+ public static Optional<ArgumentsMutator> forStaticMethod(Method method) {
+ return forStaticMethod(Mutators.newFactory(), method);
+ }
+
public static Optional<ArgumentsMutator> forInstanceMethod(
MutatorFactory mutatorFactory, Object instance, Method method) {
require(!isStatic(method), "method must not be static");
@@ -100,7 +120,7 @@
return forMethod(mutatorFactory, null, method);
}
- private static Optional<ArgumentsMutator> forMethod(
+ public static Optional<ArgumentsMutator> forMethod(
MutatorFactory mutatorFactory, Object instance, Method method) {
require(method.getParameterCount() > 0, "Can't fuzz method without parameters: " + method);
for (AnnotatedType parameter : method.getAnnotatedParameterTypes()) {
@@ -189,6 +209,10 @@
}
}
+ public Object[] getArguments() {
+ return Arrays.copyOf(arguments, arguments.length);
+ }
+
@Override
public String toString() {
return "Arguments" + productMutator;
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
index 8795bbc..addecff 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
+++ b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
@@ -242,3 +242,28 @@
"@maven//:org_junit_platform_junit_platform_testkit",
],
)
+
+[
+ java_test(
+ name = "MutatorTest" + JAZZER_FUZZ,
+ srcs = ["MutatorTest.java"],
+ env = {
+ "JAZZER_FUZZ": JAZZER_FUZZ,
+ },
+ test_class = "com.code_intelligence.jazzer.junit.MutatorTest",
+ runtime_deps = [
+ "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar",
+ "@maven//:org_junit_jupiter_junit_jupiter_engine",
+ ],
+ deps = [
+ "//src/main/java/com/code_intelligence/jazzer/api:hooks",
+ "@maven//:junit_junit",
+ "@maven//:org_junit_platform_junit_platform_engine",
+ "@maven//:org_junit_platform_junit_platform_testkit",
+ ],
+ )
+ for JAZZER_FUZZ in [
+ "",
+ "_fuzzing",
+ ]
+]
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java b/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java
new file mode 100644
index 0000000..863c57f
--- /dev/null
+++ b/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java
@@ -0,0 +1,126 @@
+// Copyright 2022 Code Intelligence GmbH
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.code_intelligence.jazzer.junit;
+
+import static org.junit.Assume.assumeFalse;
+import static org.junit.Assume.assumeTrue;
+import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
+import static org.junit.platform.testkit.engine.EventConditions.container;
+import static org.junit.platform.testkit.engine.EventConditions.displayName;
+import static org.junit.platform.testkit.engine.EventConditions.event;
+import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully;
+import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
+import static org.junit.platform.testkit.engine.EventConditions.test;
+import static org.junit.platform.testkit.engine.EventConditions.type;
+import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings;
+import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED;
+import static org.junit.platform.testkit.engine.EventType.FINISHED;
+import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED;
+import static org.junit.platform.testkit.engine.EventType.STARTED;
+import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.platform.testkit.engine.EngineExecutionResults;
+import org.junit.platform.testkit.engine.EngineTestKit;
+import org.junit.rules.TemporaryFolder;
+
+public class MutatorTest {
+ private static final String ENGINE = "engine:junit-jupiter";
+ private static final String CLASS_NAME = "com.example.MutatorFuzzTest";
+ private static final String CLAZZ = "class:" + CLASS_NAME;
+ private static final String LIFECYCLE_FUZZ = "test-template:mutatorFuzz(java.util.List)";
+ private static final String INVOCATION1 = "test-template-invocation:#1";
+ private static final String INVOCATION2 = "test-template-invocation:#2";
+
+ @Rule public TemporaryFolder temp = new TemporaryFolder();
+ private Path baseDir;
+
+ @Before
+ public void setup() throws IOException {
+ baseDir = temp.getRoot().toPath();
+ Path inputsDirectory = baseDir.resolve(
+ Paths.get("src", "test", "resources", "com", "example", "MutatorFuzzTestInputs"));
+ Files.createDirectories(inputsDirectory);
+ Files.write(inputsDirectory.resolve("invalid"), "invalid input".getBytes());
+ }
+
+ private EngineExecutionResults executeTests() {
+ System.setProperty("jazzer.experimental_mutator", "true");
+ return EngineTestKit.engine("junit-jupiter")
+ .selectors(selectClass(CLASS_NAME))
+ .configurationParameter("jazzer.instrument", "com.example.**")
+ .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString())
+ .execute();
+ }
+
+ @Test
+ public void fuzzingEnabled() {
+ assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty());
+
+ EngineExecutionResults results = executeTests();
+
+ results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
+ event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
+ event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+ event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)),
+ finishedSuccessfully()),
+ event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
+ event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
+
+ results.testEvents().assertEventsMatchExactly(
+ event(
+ type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+ event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+ displayName("Fuzzing...")),
+ event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+ displayName("Fuzzing..."), finishedWithFailure(instanceOf(AssertionError.class))));
+ }
+
+ @Test
+ public void fuzzingDisabled() {
+ assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty());
+
+ EngineExecutionResults results = executeTests();
+
+ results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
+ event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
+ event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+ event(type(REPORTING_ENTRY_PUBLISHED),
+ container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+ event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+ event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
+ event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
+
+ results.testEvents().assertEventsMatchExactly(
+ event(type(DYNAMIC_TEST_REGISTERED),
+ test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1))),
+ event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+ displayName("<empty input>")),
+ event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+ displayName("<empty input>"), finishedSuccessfully()),
+ event(type(DYNAMIC_TEST_REGISTERED),
+ test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2))),
+ event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2)),
+ displayName("invalid")),
+ event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2)),
+ displayName("invalid"), finishedSuccessfully()));
+ }
+}
