Use JUnitCore to run a list of VogarTest instances
Moved functionality for running tests with a timeout into
VogarTestRunner and added VmIsUnstableException which extends
StoppedByUserException and is used to abort the test run when a
test times out and so leaves the VM in an unstable state.
Bug: 27940141
Change-Id: I99f4c44eaf9c8e4303bdd4def5bd287bf7bdafdb
diff --git a/src/vogar/target/junit/JUnitTargetRunner.java b/src/vogar/target/junit/JUnitTargetRunner.java
index bf4ed55..58104a9 100644
--- a/src/vogar/target/junit/JUnitTargetRunner.java
+++ b/src/vogar/target/junit/JUnitTargetRunner.java
@@ -16,25 +16,17 @@
package vogar.target.junit;
-import java.util.Collections;
import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
+import org.junit.runner.JUnitCore;
import org.junit.runner.Runner;
import org.junit.runner.manipulation.NoTestsRemainException;
import org.junit.runners.model.InitializationError;
-import vogar.Result;
import vogar.monitor.TargetMonitor;
import vogar.target.Profiler;
import vogar.target.SkipPastFilter;
import vogar.target.TargetRunner;
import vogar.target.TestEnvironment;
-import vogar.util.Threads;
/**
* Adapts a JUnit3 test for use by vogar.
@@ -46,12 +38,8 @@
private final TestEnvironment testEnvironment;
private final int timeoutSeconds;
- private boolean vmIsUnstable;
private final List<VogarTest> tests;
- private final ExecutorService executor = Executors.newCachedThreadPool(
- Threads.daemonThreadFactory("testrunner"));
-
public JUnitTargetRunner(TargetMonitor monitor, AtomicReference<String> skipPastReference,
TestEnvironment testEnvironment, int timeoutSeconds, List<VogarTest> tests) {
this.monitor = monitor;
@@ -62,150 +50,33 @@
}
public boolean run(Profiler profiler) {
- for (VogarTest test : tests) {
+ // Use JUnit infrastructure to run the tests.
+ Runner runner;
+ try {
+ runner = new VogarTestRunner(tests, monitor, testEnvironment, timeoutSeconds, profiler);
+ } catch (InitializationError e) {
+ throw new IllegalStateException("Could not create VogarTestRunner", e);
+ }
- // Use JUnit infrastructure to run the tests.
- Runner runner;
+ String skipPast = skipPastReference.get();
+ if (skipPast != null) {
try {
- runner = new VogarTestRunner(Collections.singletonList(test));
- } catch (InitializationError e) {
- throw new IllegalStateException("Could not create VogarTestRunner", e);
+ new SkipPastFilter(skipPastReference).apply(runner);
+ } catch (NoTestsRemainException ignored) {
+ return true;
}
+ }
- final String skipPast = skipPastReference.get();
- if (skipPast != null) {
- try {
- new SkipPastFilter(skipPastReference).apply(runner);
- } catch (NoTestsRemainException ignored) {
- continue;
- }
- }
-
- runWithTimeout(profiler, test);
-
- if (vmIsUnstable) {
- return false;
- }
+ try {
+ JUnitCore core = new JUnitCore();
+ core.run(runner);
+ } catch (VmIsUnstableException e) {
+ // If a test reports that the VM is unstable then inform the caller so that the
+ // current process can be exited abnormally which will trigger the vogar main process
+ // to rerun the tests from after the timing out test.
+ return false;
}
return true;
}
-
- /**
- * Runs the test on another thread. If the test completes before the
- * timeout, this reports the result normally. But if the test times out,
- * this reports the timeout stack trace and begins the process of killing
- * this no-longer-trustworthy process.
- */
- private void runWithTimeout(final Profiler profiler, final VogarTest test) {
- testEnvironment.reset();
- monitor.outcomeStarted(test.toString());
-
- // Start the test on a background thread.
- final AtomicReference<Thread> executingThreadReference = new AtomicReference<Thread>();
- Future<Throwable> result = executor.submit(new Callable<Throwable>() {
- public Throwable call() throws Exception {
- executingThreadReference.set(Thread.currentThread());
- try {
- if (profiler != null) {
- profiler.start();
- }
- test.run();
- return null;
- } catch (Throwable throwable) {
- return throwable;
- } finally {
- if (profiler != null) {
- profiler.stop();
- }
- }
- }
- });
-
- // Wait until either the result arrives or the test times out.
- Throwable thrown;
- try {
- thrown = timeoutSeconds == 0
- ? result.get()
- : result.get(timeoutSeconds, TimeUnit.SECONDS);
- } catch (TimeoutException e) {
- vmIsUnstable = true;
- Thread executingThread = executingThreadReference.get();
- if (executingThread != null) {
- executingThread.interrupt();
- e.setStackTrace(executingThread.getStackTrace());
- }
- thrown = e;
- } catch (Exception e) {
- thrown = e;
- }
-
- if (thrown != null) {
- prepareForDisplay(thrown);
- thrown.printStackTrace(System.out);
- monitor.outcomeFinished(Result.EXEC_FAILED);
- } else {
- monitor.outcomeFinished(Result.SUCCESS);
- }
- }
-
- /**
- * Strip vogar's lines from the stack trace. For example, we'd strip the
- * first two Assert lines and everything after the testFoo() line in this
- * stack trace:
- *
- * at junit.framework.Assert.fail(Assert.java:198)
- * at junit.framework.Assert.assertEquals(Assert.java:56)
- * at junit.framework.Assert.assertEquals(Assert.java:61)
- * at libcore.java.net.FooTest.testFoo(FooTest.java:124)
- * at java.lang.reflect.Method.invokeNative(Native Method)
- * at java.lang.reflect.Method.invoke(Method.java:491)
- * at vogar.target.junit.Junit$JUnitTest.run(Junit.java:214)
- * at vogar.target.junit.JUnitRunner$1.call(JUnitRunner.java:112)
- * at vogar.target.junit.JUnitRunner$1.call(JUnitRunner.java:105)
- * at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:305)
- * at java.util.concurrent.FutureTask.run(FutureTask.java:137)
- * at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
- * at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:569)
- * at java.lang.Thread.run(Thread.java:863)
- */
- private void prepareForDisplay(Throwable t) {
- StackTraceElement[] stackTraceElements = t.getStackTrace();
- boolean foundVogar = false;
-
- int last = stackTraceElements.length - 1;
- for (; last >= 0; last--) {
- String className = stackTraceElements[last].getClassName();
- if (className.startsWith("vogar.target")) {
- foundVogar = true;
- } else if (foundVogar
- && !className.startsWith("java.lang.reflect")
- && !className.startsWith("sun.reflect")
- && !className.startsWith("junit.framework")) {
- if (last < stackTraceElements.length) {
- last++;
- }
- break;
- }
- }
-
- int first = 0;
- for (; first < last; first++) {
- String className = stackTraceElements[first].getClassName();
- if (!className.startsWith("junit.framework")) {
- break;
- }
- }
-
- if (first > 0) {
- first--; // retain one assertSomething() line in the trace
- }
-
- if (first < last) {
- // Arrays.copyOfRange() didn't exist on Froyo
- StackTraceElement[] copyOfRange = new StackTraceElement[last - first];
- System.arraycopy(stackTraceElements, first, copyOfRange, 0, last - first);
- t.setStackTrace(copyOfRange);
- }
- }
}
diff --git a/src/vogar/target/junit/VmIsUnstableException.java b/src/vogar/target/junit/VmIsUnstableException.java
new file mode 100644
index 0000000..99a1728
--- /dev/null
+++ b/src/vogar/target/junit/VmIsUnstableException.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vogar.target.junit;
+
+import org.junit.runner.notification.StoppedByUserException;
+
+/**
+ * A special exception used to abort a test run due to the VM becoming unstable.
+ */
+public class VmIsUnstableException extends StoppedByUserException {
+}
diff --git a/src/vogar/target/junit/VogarTestRunner.java b/src/vogar/target/junit/VogarTestRunner.java
index b5c8a24..e9a6169 100644
--- a/src/vogar/target/junit/VogarTestRunner.java
+++ b/src/vogar/target/junit/VogarTestRunner.java
@@ -17,11 +17,24 @@
package vogar.target.junit;
import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
import org.junit.runner.Description;
import org.junit.runner.notification.RunNotifier;
import org.junit.runners.ParentRunner;
import org.junit.runners.model.InitializationError;
import org.junit.runners.model.Statement;
+import vogar.Result;
+import vogar.monitor.TargetMonitor;
+import vogar.target.Profiler;
+import vogar.target.TestEnvironment;
+import vogar.util.Threads;
/**
* A {@link org.junit.runner.Runner} that can run a list of {@link VogarTest} instances.
@@ -29,10 +42,27 @@
public class VogarTestRunner extends ParentRunner<VogarTest> {
private final List<VogarTest> children;
+ private final TargetMonitor monitor;
- public VogarTestRunner(List<VogarTest> children) throws InitializationError {
+ private final TestEnvironment testEnvironment;
+ private final int timeoutSeconds;
+
+ private final ExecutorService executor = Executors.newCachedThreadPool(
+ Threads.daemonThreadFactory("testrunner"));
+ private final Profiler profiler;
+
+ private boolean vmIsUnstable;
+
+ public VogarTestRunner(List<VogarTest> children, TargetMonitor monitor,
+ TestEnvironment testEnvironment, int timeoutSeconds,
+ Profiler profiler)
+ throws InitializationError {
super(VogarTestRunner.class);
this.children = children;
+ this.monitor = monitor;
+ this.testEnvironment = testEnvironment;
+ this.timeoutSeconds = timeoutSeconds;
+ this.profiler = profiler;
}
@Override
@@ -50,8 +80,140 @@
runLeaf(new Statement() {
@Override
public void evaluate() throws Throwable {
- child.run();
+ runWithTimeout(child);
}
}, describeChild(child), notifier);
+
+ // Abort the test run if the VM is deemed unstable, i.e. the previous test timed out.
+ // Throw this after the results of the previous test have been reported.
+ if (vmIsUnstable) {
+ throw new VmIsUnstableException();
+ }
+ }
+
+ /**
+ * Runs the test on another thread. If the test completes before the
+ * timeout, this reports the result normally. But if the test times out,
+ * this reports the timeout stack trace and begins the process of killing
+ * this no-longer-trustworthy process.
+ */
+ private void runWithTimeout(final VogarTest test) {
+ testEnvironment.reset();
+ monitor.outcomeStarted(test.toString());
+
+ // Start the test on a background thread.
+ final AtomicReference<Thread> executingThreadReference = new AtomicReference<>();
+ Future<Throwable> result = executor.submit(new Callable<Throwable>() {
+ public Throwable call() throws Exception {
+ executingThreadReference.set(Thread.currentThread());
+ try {
+ if (profiler != null) {
+ profiler.start();
+ }
+ test.run();
+ return null;
+ } catch (Throwable throwable) {
+ return throwable;
+ } finally {
+ if (profiler != null) {
+ profiler.stop();
+ }
+ }
+ }
+ });
+
+ // Wait until either the result arrives or the test times out.
+ Throwable thrown;
+ try {
+ thrown = getThrowable(result);
+ } catch (TimeoutException e) {
+ vmIsUnstable = true;
+ Thread executingThread = executingThreadReference.get();
+ if (executingThread != null) {
+ executingThread.interrupt();
+ e.setStackTrace(executingThread.getStackTrace());
+ }
+ thrown = e;
+ } catch (Exception e) {
+ thrown = e;
+ }
+
+ if (thrown != null) {
+ prepareForDisplay(thrown);
+ thrown.printStackTrace(System.out);
+ monitor.outcomeFinished(Result.EXEC_FAILED);
+ } else {
+ monitor.outcomeFinished(Result.SUCCESS);
+ }
+ }
+
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ private Throwable getThrowable(Future<Throwable> result)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ Throwable thrown;
+ thrown = timeoutSeconds == 0
+ ? result.get()
+ : result.get(timeoutSeconds, TimeUnit.SECONDS);
+ return thrown;
+ }
+
+ /**
+ * Strip vogar's lines from the stack trace. For example, we'd strip the
+ * first two Assert lines and everything after the testFoo() line in this
+ * stack trace:
+ *
+ * at junit.framework.Assert.fail(Assert.java:198)
+ * at junit.framework.Assert.assertEquals(Assert.java:56)
+ * at junit.framework.Assert.assertEquals(Assert.java:61)
+ * at libcore.java.net.FooTest.testFoo(FooTest.java:124)
+ * at java.lang.reflect.Method.invokeNative(Native Method)
+ * at java.lang.reflect.Method.invoke(Method.java:491)
+ * at vogar.target.junit.Junit$JUnitTest.run(Junit.java:214)
+ * at vogar.target.junit.JUnitRunner$1.call(JUnitRunner.java:112)
+ * at vogar.target.junit.JUnitRunner$1.call(JUnitRunner.java:105)
+ * at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:305)
+ * at java.util.concurrent.FutureTask.run(FutureTask.java:137)
+ * at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
+ * at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:569)
+ * at java.lang.Thread.run(Thread.java:863)
+ */
+ public void prepareForDisplay(Throwable t) {
+ StackTraceElement[] stackTraceElements = t.getStackTrace();
+ boolean foundVogar = false;
+
+ int last = stackTraceElements.length - 1;
+ for (; last >= 0; last--) {
+ String className = stackTraceElements[last].getClassName();
+ if (className.startsWith("vogar.target")) {
+ foundVogar = true;
+ } else if (foundVogar
+ && !className.startsWith("java.lang.reflect")
+ && !className.startsWith("sun.reflect")
+ && !className.startsWith("junit.framework")) {
+ if (last < stackTraceElements.length) {
+ last++;
+ }
+ break;
+ }
+ }
+
+ int first = 0;
+ for (; first < last; first++) {
+ String className = stackTraceElements[first].getClassName();
+ if (!className.startsWith("junit.framework")) {
+ break;
+ }
+ }
+
+ if (first > 0) {
+ first--; // retain one assertSomething() line in the trace
+ }
+
+ if (first < last) {
+ // Arrays.copyOfRange() didn't exist on Froyo
+ StackTraceElement[] copyOfRange = new StackTraceElement[last - first];
+ System.arraycopy(stackTraceElements, first, copyOfRange, 0, last - first);
+ t.setStackTrace(copyOfRange);
+ }
}
}
