Use JUnit classes for running JUnit4 tests
Removes the custom code for handling JUnit4 classes and switches
to use standard JUnit classes, albeit heavily customised.
A couple of changes to the tests:
* TestRunnerJUnit4Test#testRunner_SuiteTest
The order is different from previous version of Vogar as that
sorted a flattened list but JUnit has it organized as a
hierarchy and sorts each parent's children separately so tests
which have a different parent like SimpleTest2 and SimpleTest3
are not sorted relative to each other.
* TestRunnerJUnit4Test#testRunner_AnnotatedMethodsTest
Added test because Vogar can now handle the annotations
@BeforeClass/AfterClass. It could not do that previously
because it had a flat list rather than grouping tests by
class.
* MockitoFieldTest
Added @RunWith(MockitoJUnitRunner.class) to replace the
reflective call to MockitoAnnotations.initMocks(Object) that
was previously done for all JUnit 4 tests. Presumably, Vogar
did that because it only had a partial implementation of JUnit
4 test running infrastructure and so could not support the
@RunWith mechanism.
The removal of the implicit reflective call to
MockitoAnnotations.initMocks(Object) does change the behavior
but was removed anyway because:
* There do not appear to be any tests that depend on it, at
least not in aosp/master.
* There are not many JUnit 4 tests because Vogar does not
support it very well.
* Vogar is primarily for Android developers and most of the
tests that they run they write will also be part of CTS which
requires that they do something in the test to initialize the
@Mock annotated fields, e.g. use
@RunWith(MockitoJUnitRunner.class) or explicitly call
MockitoAnnotations.initMocks(this).
* Given that Vogar now supports standard JUnit 4 it is easy to
fix any broken tests.
* It's not particularly helpful.
Testing: Build vogar-tests and run them,
run art/tools/run-libcore-tests.sh
Bug: 27940141
Change-Id: I9fd993395b52620162176a4d4c5930ded746857c
diff --git a/src/vogar/target/junit/ApplyGlobalRulesBlockJUnit4ClassRunner.java b/src/vogar/target/junit/ApplyGlobalRulesBlockJUnit4ClassRunner.java
new file mode 100644
index 0000000..1e77307
--- /dev/null
+++ b/src/vogar/target/junit/ApplyGlobalRulesBlockJUnit4ClassRunner.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import org.junit.Ignore;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runner.notification.RunNotifier;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+
+/**
+ * Applies global rules, i.e. those provided externally to the tests.
+ */
+public abstract class ApplyGlobalRulesBlockJUnit4ClassRunner extends BlockJUnit4ClassRunner {
+
+ private final TestRule testRule;
+
+ public ApplyGlobalRulesBlockJUnit4ClassRunner(Class<?> klass, TestRule testRule)
+ throws InitializationError {
+ super(klass);
+ this.testRule = testRule;
+ }
+
+ @Override
+ protected void runChild(final FrameworkMethod method, RunNotifier notifier) {
+ // Override to allow it to call abortingRunLeaf as runLeaf is final and so its behavior
+ // cannot be modified by overriding it.
+ Description description = describeChild(method);
+ if (method.getAnnotation(Ignore.class) != null) {
+ notifier.fireTestIgnored(description);
+ } else {
+ ParentRunnerHelper.abortingRunLeaf(methodBlock(method), description, notifier);
+ }
+ }
+
+ @Override
+ protected Statement methodBlock(FrameworkMethod method) {
+ // Override to apply any global TestRules.
+ Statement statement = super.methodBlock(method);
+ statement = testRule.apply(statement, getDescription());
+ return statement;
+ }
+}
diff --git a/src/vogar/target/junit/DescriptionComparator.java b/src/vogar/target/junit/DescriptionComparator.java
new file mode 100644
index 0000000..b828daf
--- /dev/null
+++ b/src/vogar/target/junit/DescriptionComparator.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import java.util.Comparator;
+import org.junit.runner.Description;
+
+/**
+ * Order by class name first, then display name.
+ */
+public class DescriptionComparator implements Comparator<Description> {
+
+ private static final DescriptionComparator DESCRIPTION_COMPARATOR =
+ new DescriptionComparator();
+
+ public static DescriptionComparator getInstance() {
+ return DESCRIPTION_COMPARATOR;
+ }
+
+ private DescriptionComparator() {
+ }
+
+ @Override
+ public int compare(Description d1, Description d2) {
+ String c1 = d1.getClassName();
+ String c2 = d2.getClassName();
+ int result = c1.compareTo(c2);
+ if (result != 0) {
+ return result;
+ }
+
+ return d1.getDisplayName().compareTo(d2.getDisplayName());
+ }
+}
diff --git a/src/vogar/target/junit/JUnitTargetRunner.java b/src/vogar/target/junit/JUnitTargetRunner.java
index f60e0e4..861df9e 100644
--- a/src/vogar/target/junit/JUnitTargetRunner.java
+++ b/src/vogar/target/junit/JUnitTargetRunner.java
@@ -20,6 +20,7 @@
import org.junit.runner.JUnitCore;
import org.junit.runner.Runner;
import org.junit.runner.manipulation.NoTestsRemainException;
+import org.junit.runner.manipulation.Sorter;
import org.junit.runners.model.RunnerBuilder;
import vogar.monitor.TargetMonitor;
import vogar.target.Profiler;
@@ -35,6 +36,9 @@
*/
public final class JUnitTargetRunner implements TargetRunner {
+ private static final Sorter DESCRIPTION_SORTER =
+ new Sorter(DescriptionComparator.getInstance());
+
private final TargetMonitor monitor;
private final AtomicReference<String> skipPastReference;
@@ -51,7 +55,8 @@
this.testEnvironment = testEnvironment;
this.testClass = testClass;
- runnerParams = new RunnerParams(qualification, args, timeoutSeconds);
+ TimeoutAndAbortRunRule timeoutRule = new TimeoutAndAbortRunRule(timeoutSeconds);
+ runnerParams = new RunnerParams(qualification, args, timeoutRule);
}
public boolean run(Profiler profiler) {
@@ -62,6 +67,10 @@
throw new IllegalStateException("Cannot create runner for: " + testClass.getName());
}
+ // Sort to ensure consistent ordering, that is done before applying any filters as the
+ // SkipPastFilter requires a consistent ordering across runs in order to work properly.
+ DESCRIPTION_SORTER.apply(runner);
+
String skipPast = skipPastReference.get();
if (skipPast != null) {
try {
diff --git a/src/vogar/target/junit/Junit4.java b/src/vogar/target/junit/Junit4.java
index 345c11f..38e4f4b 100644
--- a/src/vogar/target/junit/Junit4.java
+++ b/src/vogar/target/junit/Junit4.java
@@ -17,26 +17,11 @@
package vogar.target.junit;
import java.lang.annotation.Annotation;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import junit.framework.AssertionFailedError;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.runner.Description;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
import org.junit.runners.Suite;
-import org.junit.runners.Suite.SuiteClasses;
/**
* Utilities for manipulating JUnit4 tests.
@@ -44,116 +29,6 @@
public final class Junit4 {
private Junit4() {}
- /**
- * Creates lazy vogar test instances from the given test case or test
- * suite.
- *
- * @param methodNames if non-empty, this is the list of test method names.
- */
- static List<VogarTest> classToVogarTests(Class<?> testClass, Collection<String> methodNames) {
- List<VogarTest> result = new ArrayList<>();
- getSuiteMethods(result, testClass, methodNames);
- return result;
- }
-
- private static void getSuiteMethods(
- List<VogarTest> out, Class<?> testClass, Collection<String> methodNames) {
-
- // Try the suites first, if we find one then exit as suites take priority.
- boolean isJunit4TestClass = getSuiteTests(out, testClass);
- if (isJunit4TestClass) {
- return;
- }
-
- Collection<Object[]> argCollection = findParameters(testClass);
-
- /* JUnit 4.x: methods marked with @Test annotation. */
- if (methodNames.isEmpty()) {
- for (Method m : testClass.getMethods()) {
- if (!m.isAnnotationPresent(org.junit.Test.class)) continue;
-
- isJunit4TestClass = true;
-
- addTest(out, testClass, m, argCollection);
- }
- } else {
- for (String methodName : methodNames) {
- try {
- Method m = testClass.getMethod(methodName);
- if (!m.isAnnotationPresent(org.junit.Test.class)) continue;
-
- isJunit4TestClass = true;
-
- addTest(out, testClass, m, argCollection);
- } catch (final NoSuchMethodException e) {
- // Check to see if any method with the specified name exists.
- Throwable cause = null;
- for (Method m : testClass.getMethods()) {
- if (m.getName().equals(methodName)) {
- cause = new Exception("Method " + methodName
- + " should have no parameters");
- break;
- }
- }
- if (cause == null) {
- cause = new AssertionFailedError("Method \"" + methodName + "\" not found");
- }
- ConfigurationError error = new ConfigurationError(
- testClass.getName(), methodName, cause);
- out.add(error);
-
- // An error happened so just treat this as a JUnit4 test otherwise it will end
- // up producing another error below.
- isJunit4TestClass = true;
- }
- }
- }
-
- if (!isJunit4TestClass) {
- out.add(new ConfigurationError(testClass.getName(), null,
- new IllegalStateException("Not a test case: " + testClass)));
- }
- }
-
- private static void addTest(List<VogarTest> out, Class<?> testClass, Method m, Collection<Object[]> argCollection) {
- if (m.isAnnotationPresent(Ignore.class)) {
- // Ignore the test.
- } else if (m.getParameterTypes().length == 0) {
- addAllParameterizedTests(out, testClass, m, argCollection);
- } else {
- String name = m.getName();
- out.add(new ConfigurationError(testClass.getName(), name,
- new Exception("Method " + name + " should have no parameters")));
- }
- }
-
- @SuppressWarnings("unchecked")
- private static Collection<Object[]> findParameters(Class<?> testClass) {
- for (Method m : testClass.getMethods()) {
- for (Annotation a : m.getAnnotations()) {
- if (Parameters.class.isAssignableFrom(a.annotationType())) {
- try {
- return (Collection<Object[]>) m.invoke(testClass);
- } catch (Exception ignored) {
- }
- }
- }
- }
-
- return null;
- }
-
- private static void addAllParameterizedTests(List<VogarTest> out, Class<?> testClass, Method m,
- Collection<Object[]> argCollection) {
- if (argCollection == null) {
- out.add(TestMethod.create(testClass, m, null));
- } else {
- for (Object[] args : argCollection) {
- out.add(TestMethod.create(testClass, m, args));
- }
- }
- }
-
public static boolean isJunit4Test(Class<?> klass) {
boolean isTestSuite = false;
boolean hasSuiteClasses = false;
@@ -206,182 +81,4 @@
return false;
}
-
- private static boolean getSuiteTests(List<VogarTest> out, Class<?> suite) {
- boolean isSuite = false;
-
- /* Check for @RunWith(Suite.class) */
- for (Annotation a : suite.getAnnotations()) {
- if (RunWith.class.isAssignableFrom(a.annotationType())) {
- if (Suite.class.isAssignableFrom(((RunWith) a).value())) {
- isSuite = true;
- }
- break;
- }
- }
-
- if (!isSuite) {
- return false;
- }
-
- /* Extract classes to run */
- for (Annotation a : suite.getAnnotations()) {
- if (SuiteClasses.class.isAssignableFrom(a.annotationType())) {
- for (Class<?> clazz : ((SuiteClasses) a).value()) {
- getSuiteMethods(out, clazz, Collections.<String>emptySet());
- }
- }
- }
-
- return true;
- }
-
- private abstract static class VogarJUnitTest implements VogarTest {
- protected final Class<?> testClass;
- protected final Method method;
-
- protected VogarJUnitTest(Class<?> testClass, Method method) {
- this.testClass = testClass;
- this.method = method;
- }
-
- public void run() throws Throwable {
- Object testCase = getTestCase();
- Throwable failure = null;
-
- try {
- Class.forName("org.mockito.MockitoAnnotations")
- .getMethod("initMocks", Object.class)
- .invoke(null, testCase);
- } catch (Exception ignored) {
- }
-
- try {
- invokeMethodWithAnnotation(testCase, BeforeClass.class);
- invokeMethodWithAnnotation(testCase, Before.class);
- method.invoke(testCase);
- } catch (InvocationTargetException t) {
- failure = t.getCause();
- } catch (Throwable t) {
- failure = t;
- }
-
- try {
- invokeMethodWithAnnotation(testCase, After.class);
- } catch (InvocationTargetException t) {
- if (failure == null) {
- failure = t.getCause();
- }
- } catch (Throwable t) {
- if (failure == null) {
- failure = t;
- }
- }
-
- try {
- invokeMethodWithAnnotation(testCase, AfterClass.class);
- } catch (InvocationTargetException t) {
- if (failure == null) {
- failure = t.getCause();
- }
- } catch (Throwable t) {
- if (failure == null) {
- failure = t;
- }
- }
-
- if (!meetsExpectations(failure, method)) {
- if (failure == null) {
- throw new AssertionFailedError("Expected exception not thrown");
- } else {
- throw failure;
- }
- }
- }
-
- private void invokeMethodWithAnnotation(Object testCase, Class<?> annotation)
- throws IllegalAccessException, InvocationTargetException {
- for (Method m : testCase.getClass().getDeclaredMethods()) {
- for (Annotation a : m.getAnnotations()) {
- if (annotation.isAssignableFrom(a.annotationType())) {
- m.invoke(testCase);
- }
- }
- }
- }
-
- protected boolean meetsExpectations(Throwable failure, Method method) {
- Class<?> expected = null;
- for (Annotation a : method.getAnnotations()) {
- if (org.junit.Test.class.isAssignableFrom(a.annotationType())) {
- expected = ((org.junit.Test) a).expected();
- }
- }
- return expected == null || org.junit.Test.None.class.isAssignableFrom(expected)
- ? (failure == null)
- : (failure != null && expected.isAssignableFrom(failure.getClass()));
- }
-
- protected abstract Object getTestCase() throws Throwable;
-
- @Override
- public Description getDescription() {
- return Description.createTestDescription(testClass, method.getName());
- }
-
- @Override public String toString() {
- return testClass.getName() + "#" + method.getName();
- }
- }
-
- /**
- * A JUnit TestCase constructed on demand and then released.
- */
- private static class TestMethod extends VogarJUnitTest {
- private final Constructor<?> constructor;
- private final Object[] constructorArgs;
-
- private TestMethod(Class<?> testClass, Method method,
- Constructor<?> constructor, Object[] constructorArgs) {
- super(testClass, method);
- this.constructor = constructor;
- this.constructorArgs = constructorArgs;
- }
-
- public static VogarTest create(Class<?> testClass, Method method,
- Object[] constructorArgs) {
- if (constructorArgs != null) {
- for (Constructor<?> c : testClass.getConstructors()) {
- if (c.getParameterTypes().length == constructorArgs.length) {
- return new TestMethod(testClass, method, c, constructorArgs);
- }
- }
-
- return new ConfigurationError(testClass.getName(), method.getName(),
- new Exception("Parameterized test cases must have "
- + constructorArgs.length + " arg constructor"));
- }
-
- try {
- return new TestMethod(testClass, method, testClass.getConstructor(), null);
- } catch (NoSuchMethodException ignored) {
- }
- try {
- return new TestMethod(testClass, method,
- testClass.getConstructor(String.class), new Object[] { method.getName() });
- } catch (NoSuchMethodException ignored) {
- }
-
- return new ConfigurationError(testClass.getName(), method.getName(),
- new Exception("Test cases must have a no-arg or string constructor."));
- }
-
- @Override protected Object getTestCase() throws Throwable {
- try {
- return constructor.newInstance(constructorArgs);
- } catch (InvocationTargetException e) {
- throw e.getCause();
- }
- }
- }
}
diff --git a/src/vogar/target/junit/MappingAnnotatedBuilder.java b/src/vogar/target/junit/MappingAnnotatedBuilder.java
new file mode 100644
index 0000000..efe1d6a
--- /dev/null
+++ b/src/vogar/target/junit/MappingAnnotatedBuilder.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import com.google.common.base.Function;
+import org.junit.internal.builders.AnnotatedBuilder;
+import org.junit.runner.RunWith;
+import org.junit.runner.Runner;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * An {@link AnnotatedBuilder} that can map the {@link Runner} class specified in the
+ * {@link RunWith} annotation to a replacement one.
+ */
+public class MappingAnnotatedBuilder extends AnnotatedBuilder {
+
+ private final Function<Class<? extends Runner>, Class<? extends Runner>> mappingFunction;
+
+ public MappingAnnotatedBuilder(
+ RunnerBuilder suiteBuilder,
+ Function<Class<? extends Runner>, Class<? extends Runner>> mappingFunction) {
+ super(suiteBuilder);
+ this.mappingFunction = mappingFunction;
+ }
+
+ @Override
+ public Runner runnerForClass(Class<?> testClass) throws Exception {
+ RunWith runWith = testClass.getAnnotation(RunWith.class);
+ if (runWith != null) {
+ Class<? extends Runner> runnerClass = runWith.value();
+
+ runnerClass = mappingFunction.apply(runnerClass);
+ if (runnerClass != null) {
+ return buildRunner(runnerClass, testClass);
+ }
+ }
+
+ return null;
+ }
+}
diff --git a/src/vogar/target/junit/ParentRunnerHelper.java b/src/vogar/target/junit/ParentRunnerHelper.java
new file mode 100644
index 0000000..2d6a2f7
--- /dev/null
+++ b/src/vogar/target/junit/ParentRunnerHelper.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import org.junit.internal.AssumptionViolatedException;
+import org.junit.internal.runners.model.EachTestNotifier;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runner.notification.RunNotifier;
+import org.junit.runner.notification.StoppedByUserException;
+import org.junit.runners.ParentRunner;
+import org.junit.runners.model.Statement;
+
+/**
+ * Provides support for modifying the behaviour of {@link ParentRunner} implementations.
+ */
+public class ParentRunnerHelper {
+
+ private ParentRunnerHelper() {
+ }
+
+ /**
+ * Runs a {@link Statement} that represents a leaf (aka atomic) test, allowing the test or
+ * rather a {@link TestRule} to abort the whole test run.
+ */
+ public static void abortingRunLeaf(
+ Statement statement, Description description, RunNotifier notifier) {
+ EachTestNotifier eachNotifier = new EachTestNotifier(notifier, description);
+ eachNotifier.fireTestStarted();
+ try {
+ statement.evaluate();
+ } catch (AssumptionViolatedException e) {
+ eachNotifier.addFailedAssumption(e);
+ } catch (StoppedByUserException e) {
+ // Stop running any more tests.
+ notifier.pleaseStop();
+
+ // If this exception has a cause then the test failed, otherwise it passed.
+ Throwable cause = e.getCause();
+ if (cause != null) {
+ // The test failed so treat the cause as a normal failure.
+ eachNotifier.addFailure(cause);
+ }
+
+ // Propagate the exception back up to abort the test run.
+ throw e;
+ } catch (Throwable e) {
+ eachNotifier.addFailure(e);
+ } finally {
+ eachNotifier.fireTestFinished();
+ }
+ }
+}
diff --git a/src/vogar/target/junit/RunnerParams.java b/src/vogar/target/junit/RunnerParams.java
index 3b988c0..4ceee30 100644
--- a/src/vogar/target/junit/RunnerParams.java
+++ b/src/vogar/target/junit/RunnerParams.java
@@ -16,6 +16,7 @@
package vogar.target.junit;
+import org.junit.rules.TestRule;
import org.junit.runner.Runner;
import org.junit.runners.model.RunnerBuilder;
@@ -30,12 +31,12 @@
private final String qualification;
private final String[] args;
- private final int timeoutSeconds;
+ private final TestRule testRule;
- public RunnerParams(String qualification, String[] args, int timeoutSeconds) {
+ public RunnerParams(String qualification, String[] args, TestRule testRule) {
this.qualification = qualification;
this.args = args;
- this.timeoutSeconds = timeoutSeconds;
+ this.testRule = testRule;
}
public String getQualification() {
@@ -46,7 +47,7 @@
return args;
}
- public int getTimeoutSeconds() {
- return timeoutSeconds;
+ public TestRule getTestRule() {
+ return testRule;
}
}
diff --git a/src/vogar/target/junit/TimeoutAndAbortRunRule.java b/src/vogar/target/junit/TimeoutAndAbortRunRule.java
new file mode 100644
index 0000000..a2df074
--- /dev/null
+++ b/src/vogar/target/junit/TimeoutAndAbortRunRule.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import vogar.util.Threads;
+
+/**
+ * Times a test out and then aborts the test run.
+ */
+public class TimeoutAndAbortRunRule implements TestRule {
+
+ private final ExecutorService executor = Executors.newCachedThreadPool(
+ Threads.daemonThreadFactory(getClass().getName()));
+
+ private final int timeoutSeconds;
+
+ /**
+ * @param timeoutSeconds the timeout in seconds, if 0 then never times out.
+ */
+ public TimeoutAndAbortRunRule(int timeoutSeconds) {
+ this.timeoutSeconds = timeoutSeconds;
+ }
+
+ @Override
+ public Statement apply(final Statement base, Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ runWithTimeout(base);
+ }
+ };
+ }
+
+ /**
+ * Runs the test on another thread. If the test completes before the
+ * timeout, this reports the result normally. But if the test times out,
+ * this reports the timeout stack trace and begins the process of killing
+ * this no-longer-trustworthy process.
+ */
+ private void runWithTimeout(final Statement base) throws Throwable {
+ // Start the test on a background thread.
+ final AtomicReference<Thread> executingThreadReference = new AtomicReference<>();
+ Future<Throwable> result = executor.submit(new Callable<Throwable>() {
+ public Throwable call() throws Exception {
+ executingThreadReference.set(Thread.currentThread());
+ try {
+ base.evaluate();
+ return null;
+ } catch (Throwable throwable) {
+ return throwable;
+ }
+ }
+ });
+
+ // Wait until either the result arrives or the test times out.
+ Throwable thrown;
+ try {
+ thrown = getThrowable(result);
+ } catch (TimeoutException e) {
+ Thread executingThread = executingThreadReference.get();
+ if (executingThread != null) {
+ executingThread.interrupt();
+ e.setStackTrace(executingThread.getStackTrace());
+ }
+ // Wrap it in an exception that will cause the current run to be aborted.
+ thrown = new VmIsUnstableException(e);
+ }
+
+ if (thrown != null) {
+ throw thrown;
+ }
+ }
+
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ private Throwable getThrowable(Future<Throwable> result)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ Throwable thrown;
+ thrown = timeoutSeconds == 0
+ ? result.get()
+ : result.get(timeoutSeconds, TimeUnit.SECONDS);
+ return thrown;
+ }
+}
+
diff --git a/src/vogar/target/junit/ValidateTestMethodWhenRunBlockJUnit4ClassRunner.java b/src/vogar/target/junit/ValidateTestMethodWhenRunBlockJUnit4ClassRunner.java
new file mode 100644
index 0000000..0d6c6b9
--- /dev/null
+++ b/src/vogar/target/junit/ValidateTestMethodWhenRunBlockJUnit4ClassRunner.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.rules.TestRule;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+
+/**
+ * Defers the validation of test methods until just before they are run.
+ *
+ * <p>This does not throw an {@link InitializationError} during construction if a test method, i.e.
+ * one marked with {@link org.junit.Test @Test} annotation, is not valid. Instead it waits until the
+ * test is actually run until it does that. This is better than the standard JUnit behavior, where
+ * a single invalid test method would prevent all methods in the class from being run.
+ */
+public class ValidateTestMethodWhenRunBlockJUnit4ClassRunner
+ extends ApplyGlobalRulesBlockJUnit4ClassRunner {
+
+ protected ValidateTestMethodWhenRunBlockJUnit4ClassRunner(Class<?> klass, TestRule testRule)
+ throws InitializationError {
+ super(klass, testRule);
+ }
+
+ @Override
+ protected void validateTestMethods(List<Throwable> errors) {
+ // Overridden to avoid validation during initialization.
+ }
+
+ @Override
+ protected Statement methodInvoker(final FrameworkMethod frameworkMethod, Object test) {
+ // Overridden to perform validation of the method.
+ Statement statement = super.methodInvoker(frameworkMethod, test);
+
+ // Wrap the Statement that will invoke the method with one that will validate that the
+ // method is of the correct form.
+ return new ValidateMethodStatement(frameworkMethod, statement);
+ }
+
+ /**
+ * A {@link Statement} that validates the underlying {@link FrameworkMethod}
+ */
+ protected static class ValidateMethodStatement extends Statement {
+ private final FrameworkMethod frameworkMethod;
+ private final Statement methodInvoker;
+
+ public ValidateMethodStatement(FrameworkMethod frameworkMethod, Statement methodInvoker) {
+ this.frameworkMethod = frameworkMethod;
+ this.methodInvoker = methodInvoker;
+ }
+
+ @Override
+ public void evaluate() throws Throwable {
+ validateFrameworkMethod();
+ methodInvoker.evaluate();
+ }
+
+ private void validateFrameworkMethod() throws Throwable {
+ ArrayList<Throwable> errors = new ArrayList<>();
+ frameworkMethod.validatePublicVoidNoArg(false, errors);
+ if (!errors.isEmpty()) {
+ throw errors.get(0);
+ }
+ }
+ }
+}
diff --git a/src/vogar/target/junit/VmIsUnstableException.java b/src/vogar/target/junit/VmIsUnstableException.java
index 99a1728..ccaf40f 100644
--- a/src/vogar/target/junit/VmIsUnstableException.java
+++ b/src/vogar/target/junit/VmIsUnstableException.java
@@ -22,4 +22,8 @@
* A special exception used to abort a test run due to the VM becoming unstable.
*/
public class VmIsUnstableException extends StoppedByUserException {
+
+ public VmIsUnstableException(Throwable cause) {
+ initCause(cause);
+ }
}
diff --git a/src/vogar/target/junit/VogarBlockJUnit4ClassRunner.java b/src/vogar/target/junit/VogarBlockJUnit4ClassRunner.java
new file mode 100644
index 0000000..1fe96c8
--- /dev/null
+++ b/src/vogar/target/junit/VogarBlockJUnit4ClassRunner.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import junit.framework.AssertionFailedError;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * A specialization of {@link BlockJUnit4ClassRunner} to implement behavior required by Vogar.
+ *
+ * <ol>
+ * <li>Defers validation of test methods,
+ * see {@link ValidateTestMethodWhenRunBlockJUnit4ClassRunner}.</li>
+ * <li>Applies global rules, see {@link ApplyGlobalRulesBlockJUnit4ClassRunner}</li>
+ * <li>Selects either explicitly requested methods, or all methods.</li>
+ * </ol>
+ */
+public class VogarBlockJUnit4ClassRunner
+ extends ValidateTestMethodWhenRunBlockJUnit4ClassRunner {
+
+ private final RunnerParams runnerParams;
+
+ /**
+ * Used by annotation runner.
+ */
+ @SuppressWarnings("unused")
+ public VogarBlockJUnit4ClassRunner(Class<?> klass, RunnerBuilder suiteBuilder)
+ throws InitializationError {
+ this(klass, ((VogarRunnerBuilder) suiteBuilder).getRunnerParams());
+ }
+
+ public VogarBlockJUnit4ClassRunner(Class<?> klass, RunnerParams runnerParams)
+ throws InitializationError {
+ super(klass, runnerParams.getTestRule());
+ this.runnerParams = runnerParams;
+ }
+
+ @Override
+ protected List<FrameworkMethod> getChildren() {
+ // Overridden to handle requested methods.
+ Set<String> requestedMethodNames = JUnitUtils.mergeQualificationAndArgs(
+ runnerParams.getQualification(), runnerParams.getArgs());
+ List<FrameworkMethod> methods = super.getChildren();
+
+ // If specific methods have been requested then select them from all the methods that were
+ // found. If they cannot be found then add a fake one that will report the method as
+ // missing.
+ if (!requestedMethodNames.isEmpty()) {
+ // Store all the methods in a map by name. That should be safe as test methods do not
+ // have parameters so there can only be one method in a class with each name.
+ Map<String, FrameworkMethod> map = new HashMap<>();
+ for (FrameworkMethod method : methods) {
+ map.put(method.getName(), method);
+ }
+
+ methods = new ArrayList<>();
+ for (final String name : requestedMethodNames) {
+ FrameworkMethod method = map.get(name);
+ if (method == null) {
+ // The method could not be found so add one that when invoked will report the
+ // method as missing.
+ methods.add(new MissingFrameworkMethod(name));
+ } else {
+ methods.add(method);
+ }
+ }
+ }
+ return methods;
+ }
+
+ /**
+ * A {@link FrameworkMethod} that is used when a specific method has been requested but no
+ * suitable {@link Method} exists.
+ *
+ * <p>It overrides a number of methods that are called during normal processing in order to
+ * avoid throwing a NPE. It also overrides {@link #validatePublicVoidNoArg(boolean, List)} to
+ * report the method as being missing. It relies on a {@link ValidateMethodStatement} to call
+ * that method immediately prior to invoking the method.
+ */
+ private static class MissingFrameworkMethod extends FrameworkMethod {
+ private static final Annotation[] NO_ANNOTATIONS = new Annotation[0];
+ private final String name;
+
+ public MissingFrameworkMethod(String name) {
+ super(null);
+ this.name = name;
+ }
+
+ @Override
+ public String getName() {
+ // Overridden to avoid NPE.
+ return name;
+ }
+
+ @Override
+ public void validatePublicVoidNoArg(boolean isStatic, List<Throwable> errors) {
+ // Overridden to report the method as missing.
+ errors.add(new AssertionFailedError("Method \"" + name + "\" not found"));
+ }
+
+ @Override
+ public Annotation[] getAnnotations() {
+ // Overridden to avoid NPE.
+ return NO_ANNOTATIONS;
+ }
+
+ @Override
+ public <T extends Annotation> T getAnnotation(Class<T> annotationType) {
+ // Overridden to avoid NPE.
+ return null;
+ }
+ }
+}
diff --git a/src/vogar/target/junit/VogarJUnit4Builder.java b/src/vogar/target/junit/VogarJUnit4Builder.java
new file mode 100644
index 0000000..a0c40fb
--- /dev/null
+++ b/src/vogar/target/junit/VogarJUnit4Builder.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import org.junit.runner.Runner;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * A {@link RunnerBuilder} that always returns a {@link VogarBlockJUnit4ClassRunner}.
+ */
+class VogarJUnit4Builder extends RunnerBuilder {
+ private final VogarRunnerBuilder topRunnerBuilder;
+
+ public VogarJUnit4Builder(VogarRunnerBuilder topRunnerBuilder) {
+ this.topRunnerBuilder = topRunnerBuilder;
+ }
+
+ @Override
+ public Runner runnerForClass(Class<?> testClass) throws Throwable {
+ return new VogarBlockJUnit4ClassRunner(testClass, topRunnerBuilder);
+ }
+}
diff --git a/src/vogar/target/junit/VogarRunnerBuilder.java b/src/vogar/target/junit/VogarRunnerBuilder.java
index 4fbddaf..6e2ebc7 100644
--- a/src/vogar/target/junit/VogarRunnerBuilder.java
+++ b/src/vogar/target/junit/VogarRunnerBuilder.java
@@ -16,11 +16,14 @@
package vogar.target.junit;
+import com.google.common.base.Function;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import org.junit.runner.Runner;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.JUnit4;
import org.junit.runners.model.RunnerBuilder;
/**
@@ -29,10 +32,15 @@
*/
public class VogarRunnerBuilder extends RunnerBuilder {
+ private static final ReplaceRunnerFunction MAPPING_FUNCTION = new ReplaceRunnerFunction();
+
+ private final RunnerParams runnerParams;
private final Collection<RunnerBuilder> builders;
public VogarRunnerBuilder(RunnerParams runnerParams) {
+ this.runnerParams = runnerParams;
builders = new ArrayList<>();
+ builders.add(new MappingAnnotatedBuilder(this, MAPPING_FUNCTION));
builders.add(new VogarTestRunnerBuilder(runnerParams) {
@Override
public List<VogarTest> getVogarTests(Class<?> testClass, Set<String> methodNames) {
@@ -43,16 +51,11 @@
}
}
});
- builders.add(new VogarTestRunnerBuilder(runnerParams) {
- @Override
- public List<VogarTest> getVogarTests(Class<?> testClass, Set<String> methodNames) {
- if (Junit4.isJunit4Test(testClass)) {
- return Junit4.classToVogarTests(testClass, methodNames);
- } else {
- return null;
- }
- }
- });
+ builders.add(new VogarJUnit4Builder(this));
+ }
+
+ public RunnerParams getRunnerParams() {
+ return runnerParams;
}
@Override
@@ -67,4 +70,15 @@
return null;
}
+ private static class ReplaceRunnerFunction
+ implements Function<Class<? extends Runner>, Class<? extends Runner>> {
+ @Override
+ public Class<? extends Runner> apply(Class<? extends Runner> runnerClass) {
+ if (runnerClass == JUnit4.class || runnerClass == BlockJUnit4ClassRunner.class) {
+ return VogarBlockJUnit4ClassRunner.class;
+ } else {
+ return runnerClass;
+ }
+ }
+ }
}
diff --git a/src/vogar/target/junit/VogarTestRunner.java b/src/vogar/target/junit/VogarTestRunner.java
index ab2625b..3ccb99c 100644
--- a/src/vogar/target/junit/VogarTestRunner.java
+++ b/src/vogar/target/junit/VogarTestRunner.java
@@ -17,20 +17,12 @@
package vogar.target.junit;
import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicReference;
+import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runner.notification.RunNotifier;
import org.junit.runners.ParentRunner;
import org.junit.runners.model.InitializationError;
import org.junit.runners.model.Statement;
-import vogar.util.Threads;
/**
* A {@link org.junit.runner.Runner} that can run a list of {@link VogarTest} instances.
@@ -39,18 +31,13 @@
private final List<VogarTest> children;
- private final int timeoutSeconds;
+ private final TestRule testRule;
- private final ExecutorService executor = Executors.newCachedThreadPool(
- Threads.daemonThreadFactory("testrunner"));
-
- private boolean vmIsUnstable;
-
- public VogarTestRunner(List<VogarTest> children, int timeoutSeconds)
+ public VogarTestRunner(List<VogarTest> children, TestRule testRule)
throws InitializationError {
super(VogarTestRunner.class);
this.children = children;
- this.timeoutSeconds = timeoutSeconds;
+ this.testRule = testRule;
}
@Override
@@ -65,67 +52,14 @@
@Override
protected void runChild(final VogarTest child, RunNotifier notifier) {
- runLeaf(new Statement() {
+ Description description = describeChild(child);
+ Statement statement = new Statement() {
@Override
public void evaluate() throws Throwable {
- runWithTimeout(child);
+ child.run();
}
- }, describeChild(child), notifier);
-
- // Abort the test run if the VM is deemed unstable, i.e. the previous test timed out.
- // Throw this after the results of the previous test have been reported.
- if (vmIsUnstable) {
- throw new VmIsUnstableException();
- }
- }
-
- /**
- * Runs the test on another thread. If the test completes before the
- * timeout, this reports the result normally. But if the test times out,
- * this reports the timeout stack trace and begins the process of killing
- * this no-longer-trustworthy process.
- */
- private void runWithTimeout(final VogarTest test) throws Throwable {
- // Start the test on a background thread.
- final AtomicReference<Thread> executingThreadReference = new AtomicReference<>();
- Future<Throwable> result = executor.submit(new Callable<Throwable>() {
- public Throwable call() throws Exception {
- executingThreadReference.set(Thread.currentThread());
- try {
- test.run();
- return null;
- } catch (Throwable throwable) {
- return throwable;
- }
- }
- });
-
- // Wait until either the result arrives or the test times out.
- Throwable thrown;
- try {
- thrown = getThrowable(result);
- } catch (TimeoutException e) {
- vmIsUnstable = true;
- Thread executingThread = executingThreadReference.get();
- if (executingThread != null) {
- executingThread.interrupt();
- e.setStackTrace(executingThread.getStackTrace());
- }
- thrown = e;
- }
-
- if (thrown != null) {
- throw thrown;
- }
- }
-
- @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
- private Throwable getThrowable(Future<Throwable> result)
- throws InterruptedException, ExecutionException, TimeoutException {
- Throwable thrown;
- thrown = timeoutSeconds == 0
- ? result.get()
- : result.get(timeoutSeconds, TimeUnit.SECONDS);
- return thrown;
+ };
+ statement = testRule.apply(statement, description);
+ ParentRunnerHelper.abortingRunLeaf(statement, description, notifier);
}
}
diff --git a/src/vogar/target/junit/VogarTestRunnerBuilder.java b/src/vogar/target/junit/VogarTestRunnerBuilder.java
index 29de9c4..167b74c 100644
--- a/src/vogar/target/junit/VogarTestRunnerBuilder.java
+++ b/src/vogar/target/junit/VogarTestRunnerBuilder.java
@@ -53,7 +53,7 @@
}
});
- return new VogarTestRunner(tests, runnerParams.getTimeoutSeconds());
+ return new VogarTestRunner(tests, runnerParams.getTestRule());
}
public abstract List<VogarTest> getVogarTests(Class<?> testClass, Set<String> methodNames);
diff --git a/test/vogar/target/junit4/MockitoFieldTest.java b/test/vogar/target/junit4/MockitoFieldTest.java
index e4f4bcc..7664e85 100644
--- a/test/vogar/target/junit4/MockitoFieldTest.java
+++ b/test/vogar/target/junit4/MockitoFieldTest.java
@@ -16,13 +16,16 @@
package vogar.target.junit4;
import org.junit.Test;
+import org.junit.runner.RunWith;
import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
import static org.junit.Assert.assertNotNull;
/**
* Test that fields annotated with {@link Mock @Mock} are correctly initialized with a mock object.
*/
+@RunWith(MockitoJUnitRunner.class)
public class MockitoFieldTest {
@Mock
diff --git a/test/vogar/target/junit4/TestRunnerJUnit4Test.java b/test/vogar/target/junit4/TestRunnerJUnit4Test.java
index d61c5a6..41f80d4 100644
--- a/test/vogar/target/junit4/TestRunnerJUnit4Test.java
+++ b/test/vogar/target/junit4/TestRunnerJUnit4Test.java
@@ -210,13 +210,17 @@
TestRunner runner = testRunnerRule.createTestRunner();
runner.run();
+ // The order is different from previous version of Vogar as that sorted a flattened list
+ // but JUnit has it organized as a hierarchy and sorts each level so classes which are on
+ // a separate level, like SimpleTest2 and SimpleTest3 are not sorted relative to each
+ // other.
expectedResults()
+ .forTestClass(SimpleTest3.class)
+ .success("simple")
.forTestClass(SimpleTest2.class)
.success("Simple3")
.success("simple1")
.success("simple2")
- .forTestClass(SimpleTest3.class)
- .success("simple")
.completedNormally();
}
@@ -303,6 +307,20 @@
.completedNormally();
}
+ @TestRunnerProperties(testClass = AnnotatedMethodsTest.class)
+ @Test
+ public void testRunner_AnnotatedMethodsTest() throws Exception {
+ TestRunner runner = testRunnerRule.createTestRunner();
+ runner.run();
+
+ expectedResults()
+ .text("Before Class\n")
+ .success("test1", "Before\nTest 1\nAfter\n")
+ .success("test2", "Before\nTest 2\nAfter\n")
+ .text("After Class\n")
+ .completedNormally();
+ }
+
@TestRunnerProperties(testClass = LazyTestCreationTest.class)
@Test
public void testRunner_LazyTestCreationTest() throws Exception {
@@ -310,8 +328,10 @@
runner.run();
expectedResults()
- .success("test1", "Creating\n")
- .success("test2", "Creating\n")
+ .text("Creating\n")
+ .success("test1")
+ .text("Creating\n")
+ .success("test2")
.completedNormally();
}
