AI 144842: am: CL 144841 am: CL 144839 First version of new test runner for the core tests
that is able to filter tests, treat known failures,
isolate tests and (experimentally) find side effects
between tests. It also make it easy to run the core
tests on the RI, in order to validate the suite.
Original author: jorgp
Merged from: //branches/cupcake/...
Original author: android-build
Automated import of CL 144842
diff --git a/libcore/luni/src/test/java/com/google/coretests/CoreTestDummy.java b/libcore/luni/src/test/java/com/google/coretests/CoreTestDummy.java
new file mode 100644
index 0000000..81510f7
--- /dev/null
+++ b/libcore/luni/src/test/java/com/google/coretests/CoreTestDummy.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import junit.framework.TestCase;
+import dalvik.annotation.AndroidOnly;
+import dalvik.annotation.BrokenTest;
+import dalvik.annotation.KnownFailure;
+import dalvik.annotation.SideEffect;
+
+/**
+ * A dummy test for testing our CoreTestRunner.
+ */
+public class CoreTestDummy extends TestCase {
+
+ @AndroidOnly("")
+ public void testAndroidOnlyPass() {
+ }
+
+ @AndroidOnly("")
+ public void testAndroidOnlyFail() {
+ fail("Oops!");
+ }
+
+ @BrokenTest("")
+ public void testBrokenTestPass() {
+ }
+
+ @BrokenTest("")
+ public void testBrokenTestFail() {
+ fail("Oops!");
+ }
+
+ @KnownFailure("")
+ public void testKnownFailurePass() {
+ }
+
+ @KnownFailure("")
+ public void testKnownFailureFail() {
+ fail("Oops!");
+ }
+
+ @SideEffect("")
+ public void testSideEffectPass() {
+ }
+
+ @SideEffect("")
+ public void testSideEffectFail() {
+ fail("Oops!");
+ }
+
+ public void testNormalPass() {
+ }
+
+ public void testNormalFail() {
+ fail("Oops!");
+ }
+
+}
diff --git a/libcore/luni/src/test/java/com/google/coretests/CoreTestIsolator.java b/libcore/luni/src/test/java/com/google/coretests/CoreTestIsolator.java
new file mode 100644
index 0000000..a8bbf00
--- /dev/null
+++ b/libcore/luni/src/test/java/com/google/coretests/CoreTestIsolator.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import junit.framework.Test;
+import junit.framework.TestFailure;
+import junit.framework.TestResult;
+import junit.framework.TestSuite;
+import junit.textui.TestRunner;
+
+/**
+ * A minimalistic TestRunner implementation that silently executes a single test
+ * method and writes a possible stack trace to a temporary file. Used for
+ * isolating tests.
+ */
+public class CoreTestIsolator extends TestRunner {
+
+ /**
+ * Creates a new CoreTestIsolator. The superclass needs to be able to build
+ * a proper ResultPrinter, so we pass it a null device for printing stuff.
+ */
+ public CoreTestIsolator() {
+ super(new PrintStream(new OutputStream() {
+ @Override
+ public void write(int oneByte) throws IOException {
+ // Null device
+ }
+ }));
+ }
+
+ @Override
+ protected TestResult createTestResult() {
+ return new TestResult();
+ }
+
+ /**
+ * Provides the main entry point. First and second argument are class and
+ * method name, respectively. Third argument is the temporary file name for
+ * the result. Exits with one of the usual JUnit exit values.
+ */
+ public static void main(String args[]) {
+ Logger.global.setLevel(Level.OFF);
+
+ CoreTestIsolator testRunner = new CoreTestIsolator();
+ try {
+ TestResult r = testRunner.start(args);
+
+ if (!r.wasSuccessful()) {
+ // Store failure or error - we know there must be one
+ Throwable failure = r.failureCount() != 0 ?
+ ((TestFailure)r.failures().nextElement()).
+ thrownException() :
+ ((TestFailure)r.errors().nextElement()).
+ thrownException();
+
+ saveStackTrace(failure, args[2]);
+
+ System.exit(FAILURE_EXIT);
+ } else {
+ // Nothing to see here, please get along
+ System.exit(SUCCESS_EXIT);
+ }
+ } catch(Exception e) {
+ // Let main TestRunner know about execution problem
+ saveStackTrace(e, args[2]);
+ System.exit(EXCEPTION_EXIT);
+ }
+
+ }
+
+ /**
+ * Saves a given stack trace to a given file.
+ */
+ private static void saveStackTrace(Throwable throwable, String fileName) {
+ try {
+ FileOutputStream fos = new FileOutputStream(fileName);
+ ObjectOutputStream oos = new ObjectOutputStream(fos);
+
+ oos.writeObject(throwable);
+
+ oos.flush();
+ oos.close();
+ } catch (IOException ex) {
+ // Ignore
+ }
+ }
+
+ @Override
+ protected TestResult start(String args[]) {
+ try {
+ Test suite = TestSuite.createTest(Class.forName(args[0]), args[1]);
+ return doRun(suite);
+ }
+ catch(Exception e) {
+ throw new RuntimeException("Unable to launch test", e);
+ }
+ }
+
+}
diff --git a/libcore/luni/src/test/java/com/google/coretests/CoreTestPrinter.java b/libcore/luni/src/test/java/com/google/coretests/CoreTestPrinter.java
new file mode 100644
index 0000000..3028e24
--- /dev/null
+++ b/libcore/luni/src/test/java/com/google/coretests/CoreTestPrinter.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.io.PrintStream;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import junit.textui.ResultPrinter;
+
+/**
+ * A special ResultPrinter implementation that displays additional statistics
+ * about the test that have been executed.
+ */
+public class CoreTestPrinter extends ResultPrinter {
+
+ /**
+ * The last test class we executed.
+ */
+ private Class<?> fLastClass;
+
+ /**
+ * The current output column for dots.
+ */
+ private int fColumn;
+
+ /**
+ * The time it took to execute the tests.
+ */
+ private int fRunTime;
+
+ /**
+ * The flags the user specified.
+ */
+ private int fFlags;
+
+ /**
+ * Creates a new CoreTestPrinter for the given parameters.
+ */
+ public CoreTestPrinter(PrintStream writer, int flags) {
+ super(writer);
+ fFlags = flags;
+ }
+
+ @Override
+ protected void printHeader(long runTime) {
+ fRunTime = (int)(runTime / 1000);
+
+ if (fColumn != 0) {
+ getWriter().println();
+ }
+
+ getWriter().println();
+ }
+
+ @Override
+ protected void printFooter(TestResult result) {
+ CoreTestResult coreResult = (CoreTestResult)result;
+
+ PrintStream printer = getWriter();
+
+ if (fColumn != 0) {
+ printer.println();
+ }
+
+ printer.println();
+ printer.println("Total tests : " + coreResult.fTotalTestCount);
+ printer.println("Tests run : " + coreResult.runCount());
+ printer.println("Tests ignored : " + coreResult.fIgnoredCount);
+
+ printer.println();
+ printer.println("Normal tests : " + coreResult.fNormalTestCount);
+ printer.println("Android-only : " + coreResult.fAndroidOnlyCount);
+ printer.println("Broken tests : " + coreResult.fBrokenTestCount);
+ printer.println("Known failures: " + coreResult.fKnownFailureCount);
+ printer.println("Side-effects : " + coreResult.fSideEffectCount);
+
+ printMemory();
+
+ int seconds = fRunTime;
+
+ int hours = seconds / 3600;
+ seconds = seconds % 3600;
+
+ int minutes = seconds / 60;
+ seconds = seconds % 60;
+
+ String text = String.format("%02d:%02d:%02d", hours, minutes, seconds);
+
+ printer.println();
+ printer.println("Time taken : " + text);
+
+ super.printFooter(result);
+ }
+
+ /**
+ * Dumps some memory info.
+ */
+ private void printMemory() {
+ PrintStream printer = getWriter();
+ Runtime runtime = Runtime.getRuntime();
+
+ long total = runtime.totalMemory();
+ long free = runtime.freeMemory();
+ long used = total - free;
+
+ printer.println();
+ printer.println("Total memory : " + total);
+ printer.println("Used memory : " + used);
+ printer.println("Free memory : " + free);
+ }
+
+ @Override
+ public void startTest(Test test) {
+ TestCase caze = (TestCase)test;
+
+ if (fLastClass == null ||
+ caze.getClass().getPackage() != fLastClass.getPackage()) {
+
+ if (fColumn != 0) {
+ getWriter().println();
+ fColumn = 0;
+ }
+
+ getWriter().println();
+ Package pack = caze.getClass().getPackage();
+ getWriter().println(pack == null ? "Default package" :
+ pack.getName());
+ getWriter().println();
+
+ }
+
+ if ((fFlags & CoreTestSuite.VERBOSE) != 0) {
+ if (caze.getClass() != fLastClass) {
+ if (fColumn != 0) {
+ getWriter().println();
+ fColumn = 0;
+ }
+
+ String name = caze.getClass().getSimpleName().toString();
+
+ printMemory();
+ getWriter().println("Now executing : " + name);
+ getWriter().println();
+ }
+ }
+
+ getWriter().print(".");
+ if (fColumn++ >= 40) {
+ getWriter().println();
+ fColumn= 0;
+ }
+
+ fLastClass = caze.getClass();
+ }
+
+ @Override
+ public void addError(Test test, Throwable t) {
+ if (t instanceof CoreTestTimeout) {
+ getWriter().print("T");
+ } else {
+ super.addError(test, t);
+ }
+ }
+
+}
diff --git a/libcore/luni/src/test/java/com/google/coretests/CoreTestResult.java b/libcore/luni/src/test/java/com/google/coretests/CoreTestResult.java
new file mode 100644
index 0000000..9a821cd
--- /dev/null
+++ b/libcore/luni/src/test/java/com/google/coretests/CoreTestResult.java
@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.lang.reflect.Method;
+
+import junit.framework.Protectable;
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import dalvik.annotation.KnownFailure;
+import dalvik.annotation.SideEffect;
+
+/**
+ * A special TestResult implementation that is able to filter out annotated
+ * tests and handles our known failures properly (expects them to fail).
+ * Handy when running the Core Libraries tests on Android, the bare-metal
+ * Dalvik VM, or the RI.
+ */
+public class CoreTestResult extends TestResult {
+
+ /**
+ * The flags the user specified for this test run.
+ */
+ protected int fFlags;
+
+ /**
+ * The timeout the user specified for this test run.
+ */
+ protected int fTimeout;
+
+ /**
+ * The total number of tests in the original suite.
+ */
+ protected int fTotalTestCount;
+
+ /**
+ * The number of Android-only tests in the original suite.
+ */
+ protected int fAndroidOnlyCount;
+
+ /**
+ * The number of broken tests in the original suite.
+ */
+ protected int fBrokenTestCount;
+
+ /**
+ * The number of known failures in the original suite.
+ */
+ protected int fKnownFailureCount;
+
+ /**
+ * The number of side-effective tests in the original suite.
+ */
+ protected int fSideEffectCount;
+
+ /**
+ * The number of normal (non-annotated) tests in the original suite.
+ */
+ protected int fNormalTestCount;
+
+ /**
+ * The number of ignored tests, that is, the number of tests that were
+ * excluded from this suite due to their annotations.
+ */
+ protected int fIgnoredCount;
+
+ /**
+ * Creates a new CoreTestResult with the given flags and timeout.
+ */
+ public CoreTestResult(int flags, int timeout) {
+ super();
+
+ fFlags = flags;
+ fTimeout = timeout;
+ }
+
+ /**
+ * Checks whether the given TestCase method has the given annotation.
+ */
+ @SuppressWarnings("unchecked")
+ boolean hasAnnotation(TestCase test, Class clazz) {
+ try {
+ Method method = test.getClass().getMethod(test.getName());
+ return method.getAnnotation(clazz) != null;
+ } catch (Exception e) {
+ // Ignore
+ }
+
+ return false;
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void runProtected(final Test test, Protectable p) {
+ if ((fFlags & CoreTestSuite.DRY_RUN) == 0) {
+ if (test instanceof TestCase) {
+ TestCase testCase = (TestCase)test;
+
+ // Check whether we need to invert the test result (known failures)
+ boolean invert = hasAnnotation(testCase, KnownFailure.class) &&
+ (fFlags & CoreTestSuite.INVERT_KNOWN_FAILURES) != 0;
+
+ // Check whether we need to isolate the test (side effects)
+ boolean isolate = hasAnnotation(testCase, SideEffect.class) &&
+ (fFlags & CoreTestSuite.ISOLATE_NONE) == 0 ||
+ (fFlags & CoreTestSuite.ISOLATE_ALL) != 0;
+
+ CoreTestRunnable runnable = new CoreTestRunnable(
+ testCase, this, p, invert, isolate);
+
+ if (fTimeout > 0) {
+ Thread thread = new Thread(runnable);
+ thread.start();
+ try {
+ thread.join(fTimeout * 1000);
+ } catch (InterruptedException ex) {
+ // Ignored
+ }
+ if (thread.isAlive()) {
+ runnable.stop();
+ thread.stop();
+ try {
+ thread.join(fTimeout * 1000);
+ } catch (InterruptedException ex) {
+ // Ignored
+ }
+
+ addError(test, new CoreTestTimeout("Test timed out"));
+ }
+ } else {
+ runnable.run();
+ }
+ }
+ }
+ }
+
+ /**
+ * Updates the statistics in this TestResult. Called from the TestSuite,
+ * since, once the original suite has been filtered, we don't actually see
+ * these tests here anymore.
+ */
+ void updateStats(int total, int androidOnly, int broken, int knownFailure,
+ int normal, int ignored, int sideEffect) {
+
+ this.fTotalTestCount += total;
+ this.fAndroidOnlyCount += androidOnly;
+ this.fBrokenTestCount += broken;
+ this.fKnownFailureCount += knownFailure;
+ this.fNormalTestCount += normal;
+ this.fIgnoredCount += ignored;
+ this.fSideEffectCount += sideEffect;
+ }
+}
diff --git a/libcore/luni/src/test/java/com/google/coretests/CoreTestRunnable.java b/libcore/luni/src/test/java/com/google/coretests/CoreTestRunnable.java
new file mode 100644
index 0000000..ab49e47
--- /dev/null
+++ b/libcore/luni/src/test/java/com/google/coretests/CoreTestRunnable.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.ObjectInputStream;
+
+import junit.framework.AssertionFailedError;
+import junit.framework.Protectable;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import junit.textui.TestRunner;
+
+/**
+ * A wrapper around a single test that allows to execute the test either in the
+ * same thread, in a separate thread, or even in a different process.
+ */
+public class CoreTestRunnable implements Runnable {
+
+ private static boolean IS_DALVIK = "Dalvik".equals(
+ System.getProperty("java.vm.name"));
+
+ /**
+ * The test case we are supposed to run.
+ */
+ private TestCase fTest;
+
+ /**
+ * The TestResult we need to update after the run.
+ */
+ private TestResult fResult;
+
+ /**
+ * The Protectable that JUnit has created for us.
+ */
+ private Protectable fProtectable;
+
+ /**
+ * Reflects whether we need to invert the test result, which is used for
+ * treating known failures.
+ */
+ private boolean fInvert;
+
+ /**
+ * Reflects whether we need to isolate the test, which means we run it in
+ * a separate process.
+ */
+ private boolean fIsolate;
+
+ /**
+ * If we are isolating the test case, this holds the process that is running
+ * it.
+ */
+ private Process fProcess;
+
+ /**
+ * Creates a new CoreTestRunnable for the given parameters.
+ */
+ public CoreTestRunnable(TestCase test, TestResult result,
+ Protectable protectable, boolean invert, boolean isolate) {
+
+ this.fTest = test;
+ this.fProtectable = protectable;
+ this.fResult = result;
+ this.fInvert = invert;
+ this.fIsolate = isolate;
+ }
+
+ /**
+ * Executes the test and stores the results. May be run from a secondary
+ * Thread.
+ */
+ public void run() {
+ try {
+ if (fIsolate) {
+ runExternally();
+ } else {
+ runInternally();
+ }
+
+ if (fInvert) {
+ fInvert = false;
+ throw new AssertionFailedError(
+ "@KnownFailure expected to fail, but succeeded");
+ }
+ } catch (AssertionFailedError e) {
+ if (!fInvert) {
+ fResult.addFailure(fTest, e);
+ }
+ } catch (ThreadDeath e) { // don't catch ThreadDeath by accident
+ throw e;
+ } catch (Throwable e) {
+ if (!fInvert) {
+ fResult.addError(fTest, e);
+ }
+ }
+ }
+
+ /**
+ * Tells the test case to stop. Only used with isolation. We need to kill
+ * the external process in this case.
+ */
+ public void stop() {
+ if (fProcess != null) {
+ fProcess.destroy();
+ }
+ }
+
+ /**
+ * Runs the test case in the same process. This is basically what a
+ * run-of-the-mill JUnit does, except we might also do it in a secondary
+ * thread.
+ */
+ private void runInternally() throws Throwable {
+ fProtectable.protect();
+ }
+
+ /**
+ * Runs the test case in a different process. This is what we do for
+ * isolating test cases that have side effects or do suffer from them.
+ */
+ private void runExternally() throws Throwable {
+ Throwable throwable = null;
+
+ File file = File.createTempFile("isolation", ".tmp");
+
+ fProcess = Runtime.getRuntime().exec(
+ (IS_DALVIK ? "dalvikvm" : "java") +
+ " -classpath " + System.getProperty("java.class.path") +
+ " -Djava.home=" + System.getProperty("java.home") +
+ " -Duser.home=" + System.getProperty("user.home") +
+ " -Djava.io.tmpdir=" + System.getProperty("user.home") +
+ " com.google.coretests.CoreTestIsolator" +
+ " " + fTest.getClass().getName() +
+ " " + fTest.getName() +
+ " " + file.getAbsolutePath());
+
+ int result = fProcess.waitFor();
+
+ if (result != TestRunner.SUCCESS_EXIT) {
+ try {
+ FileInputStream fis = new FileInputStream(file);
+ ObjectInputStream ois = new ObjectInputStream(fis);
+ throwable = (Throwable)ois.readObject();
+ ois.close();
+ } catch (Exception ex) {
+ throwable = new RuntimeException("Error isolating test", ex);
+ }
+ }
+
+ file.delete();
+
+ if (throwable != null) {
+ throw throwable;
+ }
+ }
+
+}
diff --git a/libcore/luni/src/test/java/com/google/coretests/CoreTestRunner.java b/libcore/luni/src/test/java/com/google/coretests/CoreTestRunner.java
new file mode 100644
index 0000000..d469c86
--- /dev/null
+++ b/libcore/luni/src/test/java/com/google/coretests/CoreTestRunner.java
@@ -0,0 +1,299 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import junit.framework.Test;
+import junit.framework.TestResult;
+import junit.framework.TestSuite;
+import junit.textui.ResultPrinter;
+import junit.textui.TestRunner;
+
+/**
+ * A special TestRunner implementation that is able to filter out annotated
+ * tests and handles our known failures properly (expects them to fail).
+ * Handy when running the Core Libraries tests on Android, the bare-metal
+ * Dalvik VM, or the RI.
+ */
+public class CoreTestRunner extends TestRunner {
+
+ /**
+ * Reflects our environment.
+ */
+ private static boolean IS_DALVIK = "Dalvik".equals(
+ System.getProperty("java.vm.name"));
+
+ /**
+ * Defines the default flags for running on Dalvik.
+ */
+ private static final int DEFAULT_FLAGS_DALVIK =
+ CoreTestSuite.RUN_ANDROID_ONLY |
+ CoreTestSuite.RUN_NORMAL_TESTS |
+ CoreTestSuite.RUN_KNOWN_FAILURES |
+ CoreTestSuite.RUN_SIDE_EFFECTS |
+ CoreTestSuite.INVERT_KNOWN_FAILURES;
+
+ /**
+ * Defines the default flags for running on an RI.
+ */
+ private static final int DEFAULT_FLAGS_NON_DALVIK =
+ CoreTestSuite.RUN_NORMAL_TESTS |
+ CoreTestSuite.RUN_KNOWN_FAILURES |
+ CoreTestSuite.RUN_SIDE_EFFECTS;
+
+ /**
+ * Holds the flags specified by the user on the command line.
+ */
+ private int fFlags;
+
+ /**
+ * Holds the timeout value specified by the user on the command line.
+ */
+ private int fTimeout;
+
+ private int fStep = 1;
+
+ /**
+ * Creates a new instance of our CoreTestRunner.
+ */
+ public CoreTestRunner() {
+ super();
+ }
+
+ @Override
+ protected TestResult createTestResult() {
+ return new CoreTestResult(fFlags, fTimeout);
+ }
+
+ protected ResultPrinter createPrinter() {
+ return new CoreTestPrinter(System.out, fFlags);
+ }
+
+ /**
+ * Provides our main entry point.
+ */
+ public static void main(String args[]) {
+ Logger.global.setLevel(Level.OFF);
+
+ System.out.println(
+ "--------------------------------------------------");
+ System.out.println("Android Core Libraries Test Suite");
+ System.out.println("Version 1.0");
+ System.out.println(
+ "Copyright (c) 2009 The Android Open Source Project");
+ System.out.println("");
+
+ CoreTestRunner testRunner = new CoreTestRunner();
+ try {
+ TestResult r = testRunner.start(args);
+
+ System.out.println(
+ "--------------------------------------------------");
+
+ if (!r.wasSuccessful()) {
+ System.exit(FAILURE_EXIT);
+ } else {
+ System.exit(SUCCESS_EXIT);
+ }
+ } catch(Exception e) {
+ System.err.println(e.getMessage());
+ System.exit(EXCEPTION_EXIT);
+ }
+
+ }
+
+ @Override
+ public TestResult doRun(Test suite, boolean wait) {
+ setPrinter(createPrinter());
+
+ /*
+ * Make sure the original suite is unreachable after we have
+ * created the new one, so GC can dispose terminated tests.
+ */
+ suite = new CoreTestSuite(suite, fFlags, fStep, null);
+
+ return super.doRun(suite, wait);
+ }
+
+ /**
+ * Prints a help screen on the console.
+ */
+ private void showHelp() {
+ System.out.println("Usage: run-core-tests {<param>} <test>");
+ System.out.println();
+ System.out.println("Where <test> is a class name, optionally followed");
+ System.out.println("by \"#\" and a method name, and <param> is one of");
+ System.out.println("the following:");
+ System.out.println();
+ System.out.println(" --include-all");
+ System.out.println(" --exclude-all");
+ System.out.println(" --include-android-only");
+ System.out.println(" --exclude-android-only");
+ System.out.println(" --include-broken-tests");
+ System.out.println(" --exclude-broken-tests");
+ System.out.println(" --include-known-failures");
+ System.out.println(" --exclude-known-failures");
+ System.out.println(" --include-normal-tests");
+ System.out.println(" --exclude-normal-tests");
+ System.out.println(" --include-side-effects");
+ System.out.println(" --exclude-side-effects");
+ System.out.println();
+ System.out.println(" --known-failures-must-fail");
+ System.out.println(" --known-failures-must-pass");
+ System.out.println(" --timeout <seconds>");
+ // System.out.println(" --find-side-effect <test>");
+ System.out.println(" --isolate-all");
+ System.out.println(" --isolate-none");
+ System.out.println(" --verbose");
+ System.out.println(" --help");
+ System.out.println();
+ System.out.println("Default parameters are:");
+ System.out.println();
+
+ if (IS_DALVIK) {
+ System.out.println(" --include-android-only");
+ System.out.println(" --exclude-broken-tests");
+ System.out.println(" --include-known-failures");
+ System.out.println(" --include-normal-tests");
+ System.out.println(" --include-side-effects");
+ System.out.println(" --known-failures-must-fail");
+ } else {
+ System.out.println(" --exclude-android-only");
+ System.out.println(" --exclude-broken-tests");
+ System.out.println(" --include-known-failures");
+ System.out.println(" --include-normal-tests");
+ System.out.println(" --include-side-effects");
+ System.out.println(" --known-failures-must-pass");
+ }
+
+ System.out.println();
+ }
+
+ /**
+ * Tries to create a Test instance from a given string. The string might
+ * either specify a class only or a class plus a method name, separated by
+ * a "#".
+ */
+ private Test createTest(String testCase) throws Exception {
+ int p = testCase.indexOf("#");
+ if (p != -1) {
+ String testName = testCase.substring(p + 1);
+ testCase = testCase.substring(0, p);
+
+ return TestSuite.createTest(Class.forName(testCase), testName);
+ } else {
+ return getTest(testCase);
+ }
+
+ }
+
+ @Override
+ protected TestResult start(String args[]) throws Exception {
+ String testName = null;
+ // String victimName = null;
+
+ boolean wait = false;
+
+ if (IS_DALVIK) {
+ fFlags = DEFAULT_FLAGS_DALVIK;
+ } else {
+ fFlags = DEFAULT_FLAGS_NON_DALVIK;
+ }
+
+ for (int i= 0; i < args.length; i++) {
+ if (args[i].startsWith("--")) {
+ if (args[i].equals("--wait")) {
+ wait = true;
+ } else if (args[i].equals("--include-all")) {
+ fFlags = fFlags | CoreTestSuite.RUN_ALL_TESTS;
+ } else if (args[i].equals("--exclude-all")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_ALL_TESTS;
+ } else if (args[i].equals("--include-android-only")) {
+ fFlags = fFlags | CoreTestSuite.RUN_ANDROID_ONLY;
+ } else if (args[i].equals("--exclude-android-only")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_ANDROID_ONLY;
+ } else if (args[i].equals("--include-broken-tests")) {
+ fFlags = fFlags | CoreTestSuite.RUN_BROKEN_TESTS;
+ } else if (args[i].equals("--exclude-broken-tests")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_BROKEN_TESTS;
+ } else if (args[i].equals("--include-known-failures")) {
+ fFlags = fFlags | CoreTestSuite.RUN_KNOWN_FAILURES;
+ } else if (args[i].equals("--exclude-known-failures")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_KNOWN_FAILURES;
+ } else if (args[i].equals("--include-normal-tests")) {
+ fFlags = fFlags | CoreTestSuite.RUN_NORMAL_TESTS;
+ } else if (args[i].equals("--exclude-normal-tests")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_NORMAL_TESTS;
+ } else if (args[i].equals("--include-side-effects")) {
+ fFlags = fFlags | CoreTestSuite.RUN_SIDE_EFFECTS;
+ } else if (args[i].equals("--exclude-side-effects")) {
+ fFlags = fFlags & ~CoreTestSuite.RUN_SIDE_EFFECTS;
+ } else if (args[i].equals("--known-failures-must-fail")) {
+ fFlags = fFlags | CoreTestSuite.INVERT_KNOWN_FAILURES;
+ } else if (args[i].equals("--known-failures-must-pass")) {
+ fFlags = fFlags & ~CoreTestSuite.INVERT_KNOWN_FAILURES;
+ } else if (args[i].equals("--timeout")) {
+ fTimeout = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("--reverse")) {
+ fFlags = fFlags | CoreTestSuite.REVERSE;
+ } else if (args[i].equals("--step")) {
+ fStep = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("--isolate-all")) {
+ fFlags = (fFlags | CoreTestSuite.ISOLATE_ALL) &
+ ~CoreTestSuite.ISOLATE_NONE;
+ } else if (args[i].equals("--isolate-none")) {
+ fFlags = (fFlags | CoreTestSuite.ISOLATE_NONE) &
+ ~CoreTestSuite.ISOLATE_ALL;
+ } else if (args[i].equals("--verbose")) {
+ fFlags = fFlags | CoreTestSuite.VERBOSE;
+ // } else if (args[i].equals("--find-side-effect")) {
+ // victimName = args[++i];
+ } else if (args[i].equals("--dry-run")) {
+ fFlags = fFlags | CoreTestSuite.DRY_RUN;
+ } else if (args[i].equals("--help")) {
+ showHelp();
+ System.exit(1);
+ } else {
+ System.err.println("Unknown argument " + args[i] +
+ ", try --help");
+ System.exit(1);
+ }
+ } else {
+ testName = args[i];
+ }
+ }
+
+ if (IS_DALVIK) {
+ System.out.println("Using Dalvik VM version " +
+ System.getProperty("java.vm.version"));
+ } else {
+ System.out.println("Using Java VM version " +
+ System.getProperty("java.version"));
+ }
+ System.out.println();
+
+ try {
+ return doRun(createTest(testName), wait);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ throw new Exception("Could not create and run test suite: " + e);
+ }
+ }
+
+}
diff --git a/libcore/luni/src/test/java/com/google/coretests/CoreTestSuite.java b/libcore/luni/src/test/java/com/google/coretests/CoreTestSuite.java
new file mode 100644
index 0000000..3c9e7fa
--- /dev/null
+++ b/libcore/luni/src/test/java/com/google/coretests/CoreTestSuite.java
@@ -0,0 +1,355 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Enumeration;
+import java.util.Vector;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestFailure;
+import junit.framework.TestResult;
+import junit.framework.TestSuite;
+import dalvik.annotation.AndroidOnly;
+import dalvik.annotation.BrokenTest;
+import dalvik.annotation.KnownFailure;
+import dalvik.annotation.SideEffect;
+
+/**
+ * A special TestSuite implementation that flattens the hierarchy of a given
+ * JUnit TestSuite and removes tests after executing them. This is so the core
+ * tests actually have a chance to succeed, since they do consume quite some
+ * memory and many tests do not (have a chance to) cleanup properly after
+ * themselves. The class also implements our filtering mechanism for tests, so
+ * it becomes easy to only include or exclude tests based on their annotations
+ * (like, say, execute all Android-only tests that are not known to be broken).
+ */
+public class CoreTestSuite implements Test {
+
+ /**
+ * Include all normal tests in the suite.
+ */
+ public static final int RUN_NORMAL_TESTS = 1;
+
+ /**
+ * Include all broken tests in the suite.
+ */
+ public static final int RUN_BROKEN_TESTS = 2;
+
+ /**
+ * Include all known failures in the suite.
+ */
+ public static final int RUN_KNOWN_FAILURES = 4;
+
+ /**
+ * Include all Android-only tests in the suite.
+ */
+ public static final int RUN_ANDROID_ONLY = 8;
+
+ /**
+ * Include side-effective tests in the suite.
+ */
+ public static final int RUN_SIDE_EFFECTS = 16;
+
+ /**
+ * Include all tests in the suite.
+ */
+ public static final int RUN_ALL_TESTS =
+ RUN_NORMAL_TESTS | RUN_BROKEN_TESTS |
+ RUN_KNOWN_FAILURES | RUN_SIDE_EFFECTS | RUN_ANDROID_ONLY;
+
+ /**
+ * Special treatment for known failures: they are expected to fail, so we
+ * throw an Exception if they succeed and accept them failing.
+ */
+ public static final int INVERT_KNOWN_FAILURES = 32;
+
+ /**
+ * Run each test in its own VM.
+ */
+ public static final int ISOLATE_ALL = 64;
+
+ /**
+ * Run no test in its own VM.
+ */
+ public static final int ISOLATE_NONE = 128;
+
+ /**
+ * Be verbose.
+ */
+ public static final int VERBOSE = 256;
+
+ public static final int REVERSE = 512;
+
+ public static final int DRY_RUN = 1024;
+
+ /**
+ * The total number of tests in the original suite.
+ */
+ protected int fTotalCount;
+
+ /**
+ * The number of Android-only tests in the original suite.
+ */
+ protected int fAndroidOnlyCount;
+
+ /**
+ * The number of broken tests in the original suite.
+ */
+ protected int fBrokenCount;
+
+ /**
+ * The number of known failures in the original suite.
+ */
+ protected int fKnownFailureCount;
+
+ /**
+ * The number of side-effective tests in the original suite.
+ */
+ protected int fSideEffectCount;
+
+ /**
+ * The number of normal (non-annotated) tests in the original suite.
+ */
+ protected int fNormalCount;
+
+ /**
+ * The number of ignored tests, that is, the number of tests that were
+ * excluded from this suite due to their annotations.
+ */
+ protected int fIgnoredCount;
+
+ /**
+ * Contains the actual test cases in a reverse-ordered, flat list.
+ */
+ private Vector<Test> fTests = new Vector<Test>();
+
+ private TestCase fVictim;
+
+ private int fStep;
+
+ private int fFlags;
+
+ /**
+ * Creates a new CoreTestSuite for the given ordinary JUnit Test (which may
+ * be a TestCase or TestSuite). The CoreTestSuite will be a flattened and
+ * potentially filtered subset of the original JUnit Test. The flags
+ * determine the way we filter.
+ */
+ public CoreTestSuite(Test suite, int flags, int step, TestCase victim) {
+ super();
+
+ fStep = step;
+ addAndFlatten(suite, flags);
+ fVictim = victim;
+ fFlags = flags;
+ }
+
+ /**
+ * Adds the given ordinary JUnit Test (which may be a TestCase or TestSuite)
+ * to this CoreTestSuite. Note we are storing the tests in reverse order,
+ * so it's easier to remove a finished test from the end of the list.
+ */
+ private void addAndFlatten(Test test, int flags) {
+ if (test instanceof TestSuite) {
+ TestSuite suite = (TestSuite)test;
+
+ if ((flags & REVERSE) != 0) {
+ for (int i = suite.testCount() - 1; i >= 0; i--) {
+ addAndFlatten(suite.testAt(i), flags);
+ }
+ } else {
+ for (int i = 0; i < suite.testCount(); i++) {
+ addAndFlatten(suite.testAt(i), flags);
+ }
+ }
+ } else if (test instanceof TestCase) {
+ TestCase caze = (TestCase)test;
+ boolean ignoreMe = false;
+
+ boolean isAndroidOnly = hasAnnotation(caze,
+ AndroidOnly.class);
+ boolean isBrokenTest = hasAnnotation(caze,
+ BrokenTest.class);
+ boolean isKnownFailure = hasAnnotation(caze,
+ KnownFailure.class);
+ boolean isSideEffect = hasAnnotation(caze,
+ SideEffect.class);
+ boolean isNormalTest =
+ !(isAndroidOnly || isBrokenTest || isKnownFailure ||
+ isSideEffect);
+
+ if (isAndroidOnly) {
+ fAndroidOnlyCount++;
+ }
+
+ if (isBrokenTest) {
+ fBrokenCount++;
+ }
+
+ if (isKnownFailure) {
+ fKnownFailureCount++;
+ }
+
+ if (isNormalTest) {
+ fNormalCount++;
+ }
+
+ if (isSideEffect) {
+ fSideEffectCount++;
+ }
+
+ if ((flags & RUN_ANDROID_ONLY) == 0 && isAndroidOnly) {
+ ignoreMe = true;
+ }
+
+ if ((flags & RUN_BROKEN_TESTS) == 0 && isBrokenTest) {
+ ignoreMe = true;
+ }
+
+ if ((flags & RUN_KNOWN_FAILURES) == 0 && isKnownFailure) {
+ ignoreMe = true;
+ }
+
+ if (((flags & RUN_NORMAL_TESTS) == 0) && isNormalTest) {
+ ignoreMe = true;
+ }
+
+ if (((flags & RUN_SIDE_EFFECTS) == 0) && isSideEffect) {
+ ignoreMe = true;
+ }
+
+ this.fTotalCount++;
+
+ if (!ignoreMe) {
+ fTests.add(test);
+ } else {
+ this.fIgnoredCount++;
+ }
+ } else {
+ System.out.println("Warning: Don't know how to handle " +
+ test.getClass().getName() + " " + test.toString());
+ }
+ }
+
+ /**
+ * Checks whether the given TestCase class has the given annotation.
+ */
+ @SuppressWarnings("unchecked")
+ private boolean hasAnnotation(TestCase test, Class clazz) {
+ try {
+ Method method = test.getClass().getMethod(test.getName());
+ return method.getAnnotation(clazz) != null;
+ } catch (Exception e) {
+ // Ignore
+ }
+
+ return false;
+ }
+
+ /**
+ * Runs the tests and collects their result in a TestResult.
+ */
+ public void run(TestResult result) {
+ // Run tests
+ int i = 0;
+
+ while (fTests.size() != 0 && !result.shouldStop()) {
+ TestCase test = (TestCase)fTests.elementAt(i);
+
+ Thread.currentThread().setContextClassLoader(
+ test.getClass().getClassLoader());
+
+ test.run(result);
+
+ /*
+ if (fVictim != null) {
+ TestResult dummy = fVictim.run();
+
+ if (dummy.failureCount() != 0) {
+ result.addError(fTests.elementAt(i), new RuntimeException(
+ "Probable side effect",
+ ((TestFailure)dummy.failures().nextElement()).
+ thrownException()));
+ } else if (dummy.errorCount() != 0) {
+ result.addError(fTests.elementAt(i), new RuntimeException(
+ "Probable side effect",
+ ((TestFailure)dummy.errors().nextElement()).
+ thrownException()));
+ }
+ }
+ */
+
+ fTests.remove(i);
+
+ if (fTests.size() != 0) {
+ i = (i + fStep - 1) % fTests.size();
+ }
+
+ }
+
+ // Forward overall stats to TestResult, so ResultPrinter can see it.
+ if (result instanceof CoreTestResult) {
+ ((CoreTestResult)result).updateStats(
+ fTotalCount, fAndroidOnlyCount, fBrokenCount,
+ fKnownFailureCount, fNormalCount, fIgnoredCount,
+ fSideEffectCount);
+ }
+ }
+
+ /**
+ * Nulls all reference fields in the given test object. This method helps
+ * us with those test classes that don't have an explicit tearDown()
+ * method. Normally the garbage collector should take care of everything,
+ * but it can't hurt to support it a bit.
+ */
+ private void cleanup(TestCase test) {
+ Field[] fields = test.getClass().getDeclaredFields();
+ for (int i = 0; i < fields.length; i++) {
+ Field f = fields[i];
+ if (!f.getType().isPrimitive() &&
+ (f.getModifiers() & Modifier.STATIC) == 0) {
+ try {
+ f.setAccessible(true);
+ f.set(test, null);
+ } catch (Exception ex) {
+ // Nothing we can do about it.
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the tests as an enumeration. Note this is empty once the tests
+ * have been executed.
+ */
+ @SuppressWarnings("unchecked")
+ public Enumeration tests() {
+ return fTests.elements();
+ }
+
+ /**
+ * Returns the number of tests in the suite. Note this is zero once the
+ * tests have been executed.
+ */
+ public int countTestCases() {
+ return fTests.size();
+ }
+
+}
diff --git a/libcore/luni/src/test/java/com/google/coretests/CoreTestTimeout.java b/libcore/luni/src/test/java/com/google/coretests/CoreTestTimeout.java
new file mode 100644
index 0000000..864e4e4
--- /dev/null
+++ b/libcore/luni/src/test/java/com/google/coretests/CoreTestTimeout.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.coretests;
+
+/**
+ * A special exception for timeouts during tests. The CoreResultPrinter knows
+ * how to handle this.
+ */
+@SuppressWarnings("serial")
+public class CoreTestTimeout extends RuntimeException {
+
+ /**
+ * Creates a new instance with the given message.
+ */
+ public CoreTestTimeout(String message) {
+ super(message);
+ }
+
+}
diff --git a/libcore/luni/src/test/java/com/google/coretests/Main.java b/libcore/luni/src/test/java/com/google/coretests/Main.java
index 8dba719..1dcebc5 100644
--- a/libcore/luni/src/test/java/com/google/coretests/Main.java
+++ b/libcore/luni/src/test/java/com/google/coretests/Main.java
@@ -42,7 +42,7 @@
}
} else {
System.out.println("Running selected tests...");
- TestRunner.main(args);
+ CoreTestRunner.main(args);
}
Runtime.getRuntime().halt(0);
