src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java

// Copyright 2022 Code Intelligence GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.code_intelligence.jazzer.junit;

import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
import static org.junit.platform.testkit.engine.EventConditions.container;
import static org.junit.platform.testkit.engine.EventConditions.displayName;
import static org.junit.platform.testkit.engine.EventConditions.event;
import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully;
import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
import static org.junit.platform.testkit.engine.EventConditions.test;
import static org.junit.platform.testkit.engine.EventConditions.type;
import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings;
import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED;
import static org.junit.platform.testkit.engine.EventType.FINISHED;
import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED;
import static org.junit.platform.testkit.engine.EventType.STARTED;
import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.platform.testkit.engine.EngineExecutionResults;
import org.junit.platform.testkit.engine.EngineTestKit;
import org.junit.rules.TemporaryFolder;

public class MutatorTest {
  private static final String ENGINE = "engine:junit-jupiter";
  private static final String CLASS_NAME = "com.example.MutatorFuzzTest";
  private static final String CLAZZ = "class:" + CLASS_NAME;
  private static final String LIFECYCLE_FUZZ = "test-template:mutatorFuzz(java.util.List)";
  private static final String INVOCATION1 = "test-template-invocation:#1";
  private static final String INVOCATION2 = "test-template-invocation:#2";

  @Rule public TemporaryFolder temp = new TemporaryFolder();
  private Path baseDir;

  @Before
  public void setup() throws IOException {
    baseDir = temp.getRoot().toPath();
    Path inputsDirectory = baseDir.resolve(
        Paths.get("src", "test", "resources", "com", "example", "MutatorFuzzTestInputs"));
    Files.createDirectories(inputsDirectory);
    Files.write(inputsDirectory.resolve("invalid"), "invalid input".getBytes());
  }

  private EngineExecutionResults executeTests() {
    System.setProperty("jazzer.experimental_mutator", "true");
    return EngineTestKit.engine("junit-jupiter")
        .selectors(selectClass(CLASS_NAME))
        .configurationParameter("jazzer.instrument", "com.example.**")
        .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString())
        .execute();
  }

  @Test
  public void fuzzingEnabled() {
    assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty());

    EngineExecutionResults results = executeTests();

    results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)),
            finishedSuccessfully()),
        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
        event(type(FINISHED), container(ENGINE), finishedSuccessfully()));

    results.testEvents().assertEventsMatchExactly(
        event(
            type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
            displayName("Fuzzing...")),
        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
            displayName("Fuzzing..."), finishedWithFailure(instanceOf(AssertionError.class))));
  }

  @Test
  public void fuzzingDisabled() {
    assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty());

    EngineExecutionResults results = executeTests();

    results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
        event(type(REPORTING_ENTRY_PUBLISHED),
            container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
        event(type(FINISHED), container(ENGINE), finishedSuccessfully()));

    results.testEvents().assertEventsMatchExactly(
        event(type(DYNAMIC_TEST_REGISTERED),
            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1))),
        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
            displayName("<empty input>")),
        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
            displayName("<empty input>"), finishedSuccessfully()),
        event(type(DYNAMIC_TEST_REGISTERED),
            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2))),
        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2)),
            displayName("invalid")),
        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2)),
            displayName("invalid"), finishedSuccessfully()));
  }
}