Make XmlDefsTest and InstrumentationTest resumable.
Bug 3143680
Change-Id: I98510f2ce19fa65c7aac905206c46b0da01590bd
diff --git a/src/com/android/tradefed/testtype/InstrumentationTest.java b/src/com/android/tradefed/testtype/InstrumentationTest.java
index 342c19a..a86eab7 100644
--- a/src/com/android/tradefed/testtype/InstrumentationTest.java
+++ b/src/com/android/tradefed/testtype/InstrumentationTest.java
@@ -43,7 +43,7 @@
/**
* A Test that runs an instrumentation test package on given device.
*/
-public class InstrumentationTest extends AbstractRemoteTest implements IDeviceTest, IRemoteTest,
+public class InstrumentationTest extends AbstractRemoteTest implements IDeviceTest, IResumableTest,
ITimeoutCallback {
private static final String LOG_TAG = "InstrumentationTest";
@@ -102,6 +102,8 @@
private IRemoteAndroidTestRunner mRunner;
private Collection<ITestRunListener> mListeners;
+ private Collection<TestIdentifier> mRemainingTests = new ArrayList<TestIdentifier>();
+
/**
* {@inheritDoc}
*/
@@ -313,20 +315,82 @@
Collection<TestIdentifier> expectedTests) throws DeviceNotAvailableException {
CollectingTestListener testTracker = new CollectingTestListener();
mListeners.add(testTracker);
- mDevice.runInstrumentationTests(mRunner, mListeners);
- TestRunResult runResult = testTracker.getCurrentRunResults();
- if (runResult.isRunFailure() || !runResult.isRunComplete()) {
- // get the delta incomplete tests
- expectedTests.removeAll(runResult.getTests());
+ mRemainingTests = expectedTests;
+ try {
+ mDevice.runInstrumentationTests(mRunner, mListeners);
+ } finally {
+ calculateRemainingTests(mRemainingTests, testTracker);
+ }
+ rerunTests(listeners);
+ }
+
+ /**
+ * Rerun any <var>mRemainingTests</var> one by one
+ *
+ * @param listeners
+ * @throws DeviceNotAvailableException
+ */
+ private void rerunTests(final List<ITestInvocationListener> listeners)
+ throws DeviceNotAvailableException {
+ if (mRemainingTests.size() > 0) {
InstrumentationListTest testRerunner = new InstrumentationListTest(mPackageName,
- mRunnerName, expectedTests);
+ mRunnerName, mRemainingTests);
testRerunner.setDevice(getDevice());
testRerunner.setTestTimeout(getTestTimeout());
- testRerunner.run(listeners);
+ CollectingTestListener testTracker = new CollectingTestListener();
+ List<ITestInvocationListener> listenersCopy = new ArrayList<ITestInvocationListener>(
+ listeners);
+ listenersCopy.add(testTracker);
+ try {
+ testRerunner.run(listenersCopy);
+ } finally {
+ calculateRemainingTests(mRemainingTests, testTracker);
+ }
}
}
/**
+ * Remove the set of tests collected by testTracker from the set of expectedTests
+ *
+ * @param expectedTests
+ * @param testTracker
+ */
+ private void calculateRemainingTests(Collection<TestIdentifier> expectedTests,
+ CollectingTestListener testTracker) {
+ expectedTests.removeAll(testTracker.getCurrentRunResults().getTests());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public void testTimeout(TestIdentifier test) {
+ mRunner.cancel();
+ final String msg = String.format(TIMED_OUT_MSG, mTestTimeout);
+ for (ITestRunListener listener : mListeners) {
+ listener.testFailed(TestFailure.ERROR, test, msg);
+ listener.testRunFailed(msg);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void resume(List<ITestInvocationListener> listeners) throws DeviceNotAvailableException {
+ rerunTests(listeners);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void resume(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ List<ITestInvocationListener> list = new ArrayList<ITestInvocationListener>(1);
+ list.add(listener);
+ resume(list);
+ }
+
+ /**
* Collect the list of tests that should be executed by this test run.
* <p/>
* This will be done by executing the test run in 'logOnly' mode, and recording the list of
@@ -420,16 +484,4 @@
mRunner.cancel();
}
}
-
- /**
- * {@inheritDoc}
- */
- public void testTimeout(TestIdentifier test) {
- mRunner.cancel();
- final String msg = String.format(TIMED_OUT_MSG, mTestTimeout);
- for (ITestRunListener listener : mListeners) {
- listener.testFailed(TestFailure.ERROR, test, msg);
- listener.testRunFailed(msg);
- }
- }
}
diff --git a/src/com/android/tradefed/testtype/testdefs/XmlDefsTest.java b/src/com/android/tradefed/testtype/testdefs/XmlDefsTest.java
index ebe8ed2..64590a2 100644
--- a/src/com/android/tradefed/testtype/testdefs/XmlDefsTest.java
+++ b/src/com/android/tradefed/testtype/testdefs/XmlDefsTest.java
@@ -22,7 +22,7 @@
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.testtype.AbstractRemoteTest;
import com.android.tradefed.testtype.IDeviceTest;
-import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.IResumableTest;
import com.android.tradefed.testtype.InstrumentationTest;
import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.xml.AbstractXmlParser.ParseException;
@@ -34,6 +34,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -43,7 +44,7 @@
* The test definition files can either be one or more files on local file system, and/or one or
* more files stored on the device under test.
*/
-public class XmlDefsTest extends AbstractRemoteTest implements IDeviceTest, IRemoteTest {
+public class XmlDefsTest extends AbstractRemoteTest implements IDeviceTest, IResumableTest {
private static final String LOG_TAG = "XmlDefsTest";
@@ -78,6 +79,9 @@
description = "Send coverage target info to test listeners. Default true.")
private boolean mSendCoverage = true;
+ private List<InstrumentationTestDef> mTestDefs = null;
+ private InstrumentationTest mCurrentTest = null;
+
public XmlDefsTest() {
}
@@ -130,12 +134,23 @@
testDefFile.delete();
}
}
- for (InstrumentationTestDef def : parser.getTestDefs()) {
+ mTestDefs = new LinkedList<InstrumentationTestDef>(parser.getTestDefs());
+ doRun(listeners);
+ }
+
+ /**
+ * @param listeners
+ * @throws DeviceNotAvailableException
+ */
+ private void doRun(List<ITestInvocationListener> listeners) throws DeviceNotAvailableException {
+ while (!mTestDefs.isEmpty()) {
+ InstrumentationTestDef def = mTestDefs.remove(0);
// only run continuous for now. Consider making this configurable
if (def.isContinuous()) {
Log.d(LOG_TAG, String.format("Running test def %s on %s", def.getName(),
getDevice().getSerialNumber()));
InstrumentationTest test = createInstrumentationTest();
+ mCurrentTest = test;
test.setDevice(getDevice());
test.setPackageName(def.getPackage());
if (def.getRunner() != null) {
@@ -151,6 +166,7 @@
if (mSendCoverage && def.getCoverageTarget() != null) {
sendCoverage(def.getPackage(), def.getCoverageTarget(), listeners);
}
+ mCurrentTest = null;
}
}
}
@@ -218,4 +234,25 @@
InstrumentationTest createInstrumentationTest() {
return new InstrumentationTest();
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void resume(List<ITestInvocationListener> listeners) throws DeviceNotAvailableException {
+ if (mCurrentTest != null) {
+ mCurrentTest.resume(listeners);
+ }
+ doRun(listeners);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void resume(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ List<ITestInvocationListener> list = new ArrayList<ITestInvocationListener>(1);
+ list.add(listener);
+ resume(list);
+ }
}
diff --git a/tests/src/com/android/tradefed/testtype/InstrumentationTestTest.java b/tests/src/com/android/tradefed/testtype/InstrumentationTestTest.java
index 738acbf..9c2965e 100644
--- a/tests/src/com/android/tradefed/testtype/InstrumentationTestTest.java
+++ b/tests/src/com/android/tradefed/testtype/InstrumentationTestTest.java
@@ -268,8 +268,9 @@
}
});
// now expect third run to run remaining test
- mMockRemoteRunner.setMethodName(test2.getClassName(), test2.getTestName());
- mMockTestDevice.runInstrumentationTests(EasyMock.eq(mMockRemoteRunner),
+ // InstrumentationListTest will create its own runner, won't use mock runner
+ //mMockRemoteRunner.setMethodName(test2.getClassName(), test2.getTestName());
+ mMockTestDevice.runInstrumentationTests((IRemoteAndroidTestRunner)EasyMock.anyObject(),
(Collection<ITestRunListener>)EasyMock.anyObject());
EasyMock.expectLastCall().andDelegateTo(new StubTestDevice() {
@Override
@@ -294,10 +295,101 @@
mMockListener.testEnded(test2, emptyMap);
mMockListener.testRunEnded(1, EMPTY_STRING_MAP);
- EasyMock.replay(mMockRemoteRunner);
- EasyMock.replay(mMockTestDevice);
- EasyMock.replay(mMockListener);
+ EasyMock.replay(mMockRemoteRunner, mMockTestDevice, mMockListener);
mInstrumentationTest.run(mMockListener);
+ EasyMock.verify(mMockListener);
+ }
+
+ /**
+ * Test resuming a test run when first run is aborted due to
+ * {@link DeviceNotAvailableException}
+ */
+ @SuppressWarnings("unchecked")
+ public void testRun_resume() throws Exception {
+ final TestIdentifier test1 = new TestIdentifier("Test", "test1");
+ final TestIdentifier test2 = new TestIdentifier("Test", "test2");
+ final Map<String, String> emptyMap = Collections.emptyMap();
+ final String runErrorMsg = "error";
+
+ mInstrumentationTest.setRerunMode(true);
+ // expect log only mode run first to collect tests
+ mMockRemoteRunner.setLogOnly(true);
+ mMockRemoteRunner.addInstrumentationArg(InstrumentationTest.DELAY_MSEC_ARG,
+ Long.toString(mInstrumentationTest.getTestDelay()));
+ mMockTestDevice.runInstrumentationTests(EasyMock.eq(mMockRemoteRunner),
+ (Collection<ITestRunListener>)EasyMock.anyObject());
+ EasyMock.expectLastCall().andDelegateTo(new StubTestDevice() {
+ @Override
+ public void runInstrumentationTests(IRemoteAndroidTestRunner runner,
+ Collection<ITestRunListener> listeners) throws DeviceNotAvailableException {
+ // perform call back on listeners to show run of two tests
+ for (ITestRunListener listener : listeners) {
+ listener.testRunStarted(TEST_PACKAGE_VALUE, 2);
+ listener.testStarted(test1);
+ listener.testEnded(test1, emptyMap);
+ listener.testStarted(test2);
+ listener.testEnded(test2, emptyMap);
+ listener.testRunEnded(1, EMPTY_STRING_MAP);
+ }
+ }
+ });
+ // now expect second run with log only mode off
+ mMockRemoteRunner.setLogOnly(false);
+ mMockRemoteRunner.removeInstrumentationArg(InstrumentationTest.DELAY_MSEC_ARG);
+ mMockTestDevice.runInstrumentationTests(EasyMock.eq(mMockRemoteRunner),
+ (Collection<ITestRunListener>)EasyMock.anyObject());
+ EasyMock.expectLastCall().andDelegateTo(new StubTestDevice() {
+ @Override
+ public void runInstrumentationTests(IRemoteAndroidTestRunner runner,
+ Collection<ITestRunListener> listeners) throws DeviceNotAvailableException {
+ // perform call back on listeners to show run failed - only one test
+ for (ITestRunListener listener : listeners) {
+ listener.testRunStarted(TEST_PACKAGE_VALUE, 2);
+ listener.testStarted(test1);
+ listener.testEnded(test1, emptyMap);
+ listener.testRunFailed(runErrorMsg);
+ }
+ throw new DeviceNotAvailableException();
+ }
+ });
+
+ // now expect third run to run remaining test
+ // InstrumentationListTest will create its own runner, won't use mock runner
+ //mMockRemoteRunner.setMethodName(test2.getClassName(), test2.getTestName());
+ mMockTestDevice.runInstrumentationTests((IRemoteAndroidTestRunner)EasyMock.anyObject(),
+ (Collection<ITestRunListener>)EasyMock.anyObject());
+ EasyMock.expectLastCall().andDelegateTo(new StubTestDevice() {
+ @Override
+ public void runInstrumentationTests(IRemoteAndroidTestRunner runner,
+ Collection<ITestRunListener> listeners) throws DeviceNotAvailableException {
+ // perform call back on listeners to show run failed - only one test
+ for (ITestRunListener listener : listeners) {
+ listener.testRunStarted(TEST_PACKAGE_VALUE, 1);
+ listener.testStarted(test2);
+ listener.testEnded(test2, emptyMap);
+ listener.testRunEnded(1, EMPTY_STRING_MAP);
+ }
+ }
+ });
+
+ mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2);
+ mMockListener.testStarted(test1);
+ mMockListener.testEnded(test1, emptyMap);
+ mMockListener.testRunFailed(runErrorMsg);
+ mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 1);
+ mMockListener.testStarted(test2);
+ mMockListener.testEnded(test2, emptyMap);
+ mMockListener.testRunEnded(1, EMPTY_STRING_MAP);
+
+ EasyMock.replay(mMockRemoteRunner, mMockTestDevice, mMockListener);
+ try {
+ mInstrumentationTest.run(mMockListener);
+ fail("DeviceNotAvailableException not thrown");
+ } catch (DeviceNotAvailableException e) {
+ // expected
+ }
+ mInstrumentationTest.resume(mMockListener);
+ EasyMock.verify(mMockListener);
}
/**
diff --git a/tests/src/com/android/tradefed/testtype/MockInstrumentationTest.java b/tests/src/com/android/tradefed/testtype/MockInstrumentationTest.java
index 5b15cee..18f8f16 100644
--- a/tests/src/com/android/tradefed/testtype/MockInstrumentationTest.java
+++ b/tests/src/com/android/tradefed/testtype/MockInstrumentationTest.java
@@ -15,6 +15,7 @@
*/
package com.android.tradefed.testtype;
+import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.result.ITestInvocationListener;
import java.util.List;
@@ -25,18 +26,52 @@
public class MockInstrumentationTest extends InstrumentationTest {
private ITestInvocationListener mListener = null;
+ private DeviceNotAvailableException mException = null;
@Override
- public void run(final List<ITestInvocationListener> listeners) {
+ public void run(final List<ITestInvocationListener> listeners)
+ throws DeviceNotAvailableException {
mListener = listeners.get(0);
+ if (mException != null) {
+ throw mException;
+ }
}
@Override
- public void run(final ITestInvocationListener listener) {
+ public void run(final ITestInvocationListener listener) throws DeviceNotAvailableException {
mListener = listener;
+ if (mException != null) {
+ throw mException;
+ }
}
public ITestInvocationListener getListener() {
return mListener;
}
+
+ @Override
+ public void resume(final List<ITestInvocationListener> listeners)
+ throws DeviceNotAvailableException {
+ mListener = listeners.get(0);
+ if (mException != null) {
+ throw mException;
+ }
+ }
+
+ @Override
+ public void resume(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ mListener = listener;
+ if (mException != null) {
+ throw mException;
+ }
+
+ }
+
+ public void clearListener() {
+ mListener = null;
+ }
+
+ public void setException(DeviceNotAvailableException e) {
+ mException = e;
+ }
}
diff --git a/tests/src/com/android/tradefed/testtype/testdefs/XmlDefsTestTest.java b/tests/src/com/android/tradefed/testtype/testdefs/XmlDefsTestTest.java
index a86a2a7..4274365 100644
--- a/tests/src/com/android/tradefed/testtype/testdefs/XmlDefsTestTest.java
+++ b/tests/src/com/android/tradefed/testtype/testdefs/XmlDefsTestTest.java
@@ -73,6 +73,19 @@
public void testRun() throws DeviceNotAvailableException {
mXmlTest.addRemoteFilePath(TEST_PATH);
+ injectMockXmlData();
+ mMockListener.testRunStarted(TEST_PKG, 0);
+ Capture<Map<String, String>> captureMetrics = new Capture<Map<String, String>>();
+ mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captureMetrics));
+ EasyMock.replay(mMockTestDevice, mMockListener);
+ mXmlTest.run(mMockListener);
+ assertEquals(mMockListener, mMockInstrumentationTest.getListener());
+ assertEquals(TEST_PKG, mMockInstrumentationTest.getPackageName());
+ assertEquals(TEST_COVERAGE_TARGET, captureMetrics.getValue().get(
+ XmlDefsTest.COVERAGE_TARGET_KEY));
+ }
+
+ private void injectMockXmlData() throws DeviceNotAvailableException {
// TODO: it would be nice to mock out the file objects, so this test wouldn't need to do
// IO
mMockTestDevice.pullFile(EasyMock.eq(TEST_PATH), (File)EasyMock.anyObject());
@@ -93,15 +106,31 @@
return false;
}
});
- mMockListener.testRunStarted(TEST_PKG, 0);
- Capture<Map<String, String>> captureMetrics = new Capture<Map<String, String>>();
- mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captureMetrics));
+ }
+
+ /**
+ * Test a run that was aborted then resumed
+ */
+ @SuppressWarnings("unchecked")
+ public void testRun_resume() throws DeviceNotAvailableException {
+ mXmlTest.addRemoteFilePath(TEST_PATH);
+
+ injectMockXmlData();
+ mMockInstrumentationTest.setException(new DeviceNotAvailableException());
EasyMock.replay(mMockTestDevice, mMockListener);
- mXmlTest.run(mMockListener);
+ try {
+ mXmlTest.run(mMockListener);
+ fail("DeviceNotAvailableException not thrown");
+ } catch (DeviceNotAvailableException e) {
+ // expected
+ }
+ // verify InstrumentationTest.run was called
assertEquals(mMockListener, mMockInstrumentationTest.getListener());
- assertEquals(TEST_PKG, mMockInstrumentationTest.getPackageName());
- assertEquals(TEST_COVERAGE_TARGET, captureMetrics.getValue().get(
- XmlDefsTest.COVERAGE_TARGET_KEY));
+ mMockInstrumentationTest.setException(null);
+ mMockInstrumentationTest.clearListener();
+ mXmlTest.resume(mMockListener);
+ // verify InstrumentationTest.resume was called
+ assertEquals(mMockListener, mMockInstrumentationTest.getListener());
}
/**
