Support tracking not executed cts tests.
Also fix issue in InstrumentationApkTest where it attempts to communicate with
device after its already deemed not available.
Bug 5376439
Change-Id: I1619586629d27e80be7586ece37908dc0e50be1c
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
index 380f891..4770e92 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
@@ -16,6 +16,8 @@
package com.android.cts.tradefed.result;
+import android.tests.getinfo.DeviceInfoConstants;
+
import com.android.cts.tradefed.build.CtsBuildHelper;
import com.android.cts.tradefed.device.DeviceInfoCollector;
import com.android.cts.tradefed.testtype.CtsTest;
@@ -25,17 +27,17 @@
import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.build.IFolderBuildInfo;
import com.android.tradefed.config.Option;
+import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.CollectingTestListener;
import com.android.tradefed.result.InputStreamSource;
import com.android.tradefed.result.LogDataType;
import com.android.tradefed.result.TestResult;
import com.android.tradefed.result.TestRunResult;
import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.StreamUtil;
import org.kxml2.io.KXmlSerializer;
-import android.tests.getinfo.DeviceInfoConstants;
-
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -151,10 +153,11 @@
@Override
public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) {
super.testRunEnded(elapsedTime, runMetrics);
- Log.i(LOG_TAG, String.format("Test run %s complete. Tests passed %d, failed %d, error %d",
+ CLog.i("%s complete: Passed %d, Failed %d, Not Executed %d",
getCurrentRunResults().getName(), getCurrentRunResults().getNumPassedTests(),
- getCurrentRunResults().getNumFailedTests(),
- getCurrentRunResults().getNumErrorTests()));
+ getCurrentRunResults().getNumFailedTests() +
+ getCurrentRunResults().getNumErrorTests(),
+ getCurrentRunResults().getNumIncompleteTests());
}
/**
@@ -187,21 +190,16 @@
serializeResultsDoc(serializer, startTimestamp, endTime);
serializer.endDocument();
// TODO: output not executed timeout omitted counts
- String msg = String.format("XML test result file generated at %s. Total tests %d, " +
- "Failed %d, Error %d", getReportPath(), getNumTotalTests(),
- getNumFailedTests(), getNumErrorTests());
+ String msg = String.format("XML test result file generated at %s. Passed %d, " +
+ "Failed %d, Not Executed %d", getReportPath(), getNumPassedTests(),
+ getNumFailedTests() + getNumErrorTests(), getNumIncompleteTests());
Log.logAndDisplay(LogLevel.INFO, LOG_TAG, msg);
Log.logAndDisplay(LogLevel.INFO, LOG_TAG, String.format("Time: %s",
TimeUtil.formatElapsedTime(elapsedTime)));
} catch (IOException e) {
Log.e(LOG_TAG, "Failed to generate report data");
} finally {
- if (stream != null) {
- try {
- stream.close();
- } catch (IOException ignored) {
- }
- }
+ StreamUtil.closeStream(stream);
}
}
@@ -414,7 +412,8 @@
serializer.attribute(ns, "failed", Integer.toString(getNumErrorTests() +
getNumFailedTests()));
// TODO: output notExecuted, timeout count
- serializer.attribute(ns, "notExecuted", "0");
+ serializer.attribute(ns, "notExecuted", Integer.toString(getNumIncompleteTests()));
+ // ignore timeouts - these are reported as errors
serializer.attribute(ns, "timeout", "0");
serializer.attribute(ns, "pass", Integer.toString(getNumPassedTests()));
serializer.endTag(ns, "Summary");
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestCase.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestCase.java
index 65e00ff..3ebb6e3 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestCase.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestCase.java
@@ -15,6 +15,7 @@
*/
package com.android.cts.tradefed.result;
+import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.TestResult;
import com.android.tradefed.result.TestResult.TestStatus;
@@ -105,8 +106,10 @@
return "fail";
case PASSED:
return "pass";
- // TODO add notExecuted
+ case INCOMPLETE:
+ return "notExecuted";
}
+ CLog.w("Unrecognized status %s", status);
return "fail";
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
index cc5dd37..88f58f8 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/CtsTest.java
@@ -43,6 +43,7 @@
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
@@ -111,12 +112,12 @@
private boolean mScreenshot = false;
/** data structure for a {@link IRemoteTest} and its known tests */
- private class KnownTests {
+ class TestPackage {
private final IRemoteTest mTestForPackage;
private final Collection<TestIdentifier> mKnownTests;
private final ITestPackageDef mPackageDef;
- KnownTests(ITestPackageDef packageDef, IRemoteTest testForPackage,
+ TestPackage(ITestPackageDef packageDef, IRemoteTest testForPackage,
Collection<TestIdentifier> knownTests) {
mPackageDef = packageDef;
mTestForPackage = testForPackage;
@@ -134,10 +135,18 @@
ITestPackageDef getPackageDef() {
return mPackageDef;
}
+
+ /**
+ * Return the test run name that should be used for the TestPackage
+ * @return
+ */
+ String getTestRunName() {
+ return mPackageDef.getUri();
+ }
}
/** list of remaining tests to execute */
- private List<KnownTests> mRemainingTests = null;
+ private List<TestPackage> mRemainingTestPkgs = null;
private CtsBuildHelper mCtsBuild = null;
private IBuildInfo mBuildInfo = null;
@@ -247,38 +256,44 @@
throw new IllegalArgumentException("missing device");
}
- if (mRemainingTests == null) {
+ if (mRemainingTestPkgs == null) {
checkFields();
- mRemainingTests = buildTestsToRun();
- }
- // always collect the device info, even for resumed runs, since test will likely be running
- // on a different device
- collectDeviceInfo(getDevice(), mCtsBuild, listener);
-
- while (!mRemainingTests.isEmpty()) {
- KnownTests knownTests = mRemainingTests.get(0);
-
- IRemoteTest test = knownTests.getTestForPackage();
- if (test instanceof IDeviceTest) {
- ((IDeviceTest)test).setDevice(getDevice());
- }
- if (test instanceof IBuildReceiver) {
- ((IBuildReceiver)test).setBuild(mBuildInfo);
- }
-
- ResultFilter filter = new ResultFilter(listener, knownTests.getKnownTests());
- test.run(filter);
- forwardPackageDetails(knownTests.getPackageDef(), listener);
- mRemainingTests.remove(0);
+ mRemainingTestPkgs = buildTestsToRun();
}
- if (mScreenshot) {
- InputStreamSource screenshotSource = getDevice().getScreenshot();
- try {
- listener.testLog("screenshot", LogDataType.PNG, screenshotSource);
- } finally {
- screenshotSource.cancel();
+ ResultFilter filter = new ResultFilter(listener, mRemainingTestPkgs);
+
+ try {
+ // always collect the device info, even for resumed runs, since test will likely be
+ // running on a different device
+ collectDeviceInfo(getDevice(), mCtsBuild, listener);
+
+ while (!mRemainingTestPkgs.isEmpty()) {
+ TestPackage knownTests = mRemainingTestPkgs.get(0);
+
+ IRemoteTest test = knownTests.getTestForPackage();
+ if (test instanceof IDeviceTest) {
+ ((IDeviceTest)test).setDevice(getDevice());
+ }
+ if (test instanceof IBuildReceiver) {
+ ((IBuildReceiver)test).setBuild(mBuildInfo);
+ }
+
+ forwardPackageDetails(knownTests.getPackageDef(), listener);
+ test.run(filter);
+ mRemainingTestPkgs.remove(0);
}
+
+ if (mScreenshot) {
+ InputStreamSource screenshotSource = getDevice().getScreenshot();
+ try {
+ listener.testLog("screenshot", LogDataType.PNG, screenshotSource);
+ } finally {
+ screenshotSource.cancel();
+ }
+ }
+ } finally {
+ filter.reportUnexecutedTests();
}
}
@@ -287,8 +302,8 @@
*
* @return
*/
- private List<KnownTests> buildTestsToRun() {
- List<KnownTests> testList = new LinkedList<KnownTests>();
+ private List<TestPackage> buildTestsToRun() {
+ List<TestPackage> testList = new LinkedList<TestPackage>();
try {
ITestCaseRepo testRepo = createTestCaseRepo();
Collection<String> testUris = getTestPackageUrisToRun(testRepo);
@@ -315,14 +330,14 @@
* @param testUri
* @param testPackage
*/
- private void addTestPackage(List<KnownTests> testList, String testUri,
+ private void addTestPackage(List<TestPackage> testList, String testUri,
ITestPackageDef testPackage) {
if (testPackage != null) {
IRemoteTest testForPackage = testPackage.createTest(mCtsBuild.getTestCasesDir(),
mClassName, mMethodName);
if (testForPackage != null) {
Collection<TestIdentifier> knownTests = testPackage.getTests();
- testList.add(new KnownTests(testPackage, testForPackage, knownTests));
+ testList.add(new TestPackage(testPackage, testForPackage, knownTests));
}
} else {
Log.e(LOG_TAG, String.format("Could not find test package uri %s", testUri));
@@ -377,7 +392,7 @@
return null;
}
checkFields();
- List<KnownTests> allTests = buildTestsToRun();
+ List<TestPackage> allTests = buildTestsToRun();
if (allTests.size() <= 1) {
Log.w(LOG_TAG, "no tests to shard!");
@@ -389,13 +404,13 @@
// don't create more shards than the number of tests we have!
for (int i = 0; i < mShards && i < allTests.size(); i++) {
CtsTest shard = new CtsTest();
- shard.mRemainingTests = new LinkedList<KnownTests>();
+ shard.mRemainingTestPkgs = new LinkedList<TestPackage>();
shardQueue.add(shard);
}
while (!allTests.isEmpty()) {
- KnownTests testPair = allTests.remove(0);
+ TestPackage testPair = allTests.remove(0);
CtsTest shard = (CtsTest)shardQueue.poll();
- shard.mRemainingTests.add(testPair);
+ shard.mRemainingTestPkgs.add(testPair);
shardQueue.add(shard);
}
return shardQueue;
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/InstrumentationApkTest.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/InstrumentationApkTest.java
index e1bb91e..e911403 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/InstrumentationApkTest.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/InstrumentationApkTest.java
@@ -71,23 +71,20 @@
Assert.assertNotNull("missing device", getDevice());
Assert.assertNotNull("missing build", mCtsBuild);
- try {
- for (String apkFileName : mInstallFileNames) {
- Log.d(LOG_TAG, String.format("Installing %s on %s", apkFileName,
- getDevice().getSerialNumber()));
- try {
- getDevice().installPackage(mCtsBuild.getTestApp(apkFileName), true);
- } catch (FileNotFoundException e) {
- Log.e(LOG_TAG, e);
- }
+ for (String apkFileName : mInstallFileNames) {
+ Log.d(LOG_TAG, String.format("Installing %s on %s", apkFileName,
+ getDevice().getSerialNumber()));
+ try {
+ getDevice().installPackage(mCtsBuild.getTestApp(apkFileName), true);
+ } catch (FileNotFoundException e) {
+ Log.e(LOG_TAG, e);
}
- super.run(listener);
- } finally {
- for (String packageName : mUninstallPackages) {
- Log.d(LOG_TAG, String.format("Uninstalling %s on %s", packageName,
- getDevice().getSerialNumber()));
- getDevice().uninstallPackage(packageName);
- }
+ }
+ super.run(listener);
+ for (String packageName : mUninstallPackages) {
+ Log.d(LOG_TAG, String.format("Uninstalling %s on %s", packageName,
+ getDevice().getSerialNumber()));
+ getDevice().uninstallPackage(packageName);
}
}
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java
index 02595c0..68bb62e 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java
@@ -15,41 +15,60 @@
*/
package com.android.cts.tradefed.testtype;
-import com.android.ddmlib.Log;
+import com.android.cts.tradefed.testtype.CtsTest.TestPackage;
import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.ResultForwarder;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
/**
* A {@link ITestInvocationListener} that filters test results based on the set of expected tests
* in CTS test package xml files.
+ * <p/>
+ * It will only report test results for expected tests, and at end of invocation, will report the
+ * set of expected tests that were not executed.
*/
class ResultFilter extends ResultForwarder {
- private final Collection<TestIdentifier> mKnownTests;
+ private final Map<String, Collection<TestIdentifier>> mKnownTestsMap;
+ private final Map<String, Collection<TestIdentifier>> mRemainingTestsMap;
+ private String mCurrentTestRun = null;
/**
* Create a {@link ResultFilter}.
*
* @param listener the real {@link ITestInvocationListener} to forward results to
- * @param expectedTests the full collection of known tests to expect
*/
- ResultFilter(ITestInvocationListener listener, Collection<TestIdentifier> knownTests) {
+ ResultFilter(ITestInvocationListener listener, List<TestPackage> testPackages) {
super(listener);
- mKnownTests = knownTests;
+
+ mKnownTestsMap = new HashMap<String, Collection<TestIdentifier>>();
+ // use LinkedHashMap for predictable test order
+ mRemainingTestsMap = new LinkedHashMap<String, Collection<TestIdentifier>>();
+
+ for (TestPackage testPkg : testPackages) {
+ mKnownTestsMap.put(testPkg.getTestRunName(), new HashSet<TestIdentifier>(
+ testPkg.getKnownTests()));
+ mRemainingTestsMap.put(testPkg.getTestRunName(), new LinkedHashSet<TestIdentifier>(
+ testPkg.getKnownTests()));
+ }
}
/**
* {@inheritDoc}
*/
@Override
- public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) {
- super.testRunEnded(elapsedTime, runMetrics);
- // TODO: report all remaining tests in mTestPackage as failed tests with
- // notExecuted result
+ public void testRunStarted(String runName, int testCount) {
+ super.testRunStarted(runName, testCount);
+ mCurrentTestRun = runName;
}
/**
@@ -60,7 +79,7 @@
if (isKnownTest(test)) {
super.testStarted(test);
} else {
- Log.d("ResultFilter", String.format("Skipping reporting unknown test %s", test));
+ CLog.d("Skipping reporting unknown test %s", test);
}
}
@@ -81,6 +100,7 @@
public void testEnded(TestIdentifier test, Map<String, String> testMetrics) {
if (isKnownTest(test)) {
super.testEnded(test, testMetrics);
+ removeExecutedTest(test);
}
}
@@ -89,6 +109,35 @@
* @return
*/
private boolean isKnownTest(TestIdentifier test) {
- return mKnownTests.contains(test);
+ if (mCurrentTestRun != null && mKnownTestsMap.containsKey(mCurrentTestRun)) {
+ return mKnownTestsMap.get(mCurrentTestRun).contains(test);
+ }
+ return false;
+ }
+
+ /**
+ * Remove given test from the 'remaining tests' data structure.
+ * @param test
+ */
+ private void removeExecutedTest(TestIdentifier test) {
+ if (mCurrentTestRun != null && mRemainingTestsMap.containsKey(mCurrentTestRun)) {
+ mRemainingTestsMap.get(mCurrentTestRun).remove(test);
+ }
+ }
+
+ /**
+ * Report the set of expected tests that were not executed
+ */
+ public void reportUnexecutedTests() {
+ for (Map.Entry<String, Collection<TestIdentifier>> entry : mRemainingTestsMap.entrySet()) {
+ super.testRunStarted(entry.getKey(), entry.getValue().size());
+ for (TestIdentifier test : entry.getValue()) {
+ // an unexecuted test is currently reported as a 'testStarted' event without a
+ // 'testEnded'. TODO: consider adding an explict API for reporting an unexecuted
+ // test
+ super.testStarted(test);
+ }
+ super.testRunEnded(0, new HashMap<String,String>());
+ }
}
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
index 58f2c00..9d70a61 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/TestPackageDef.java
@@ -197,11 +197,11 @@
// a reference app test is just a InstrumentationTest with one extra apk to install
InstrumentationApkTest instrTest = new InstrumentationApkTest();
instrTest.addInstallApk(String.format("%s.apk", mApkToTestName), mPackageToTest);
- return setInstrumentationTest(className, methodName, instrTest, testCaseDir);
+ return setInstrumentationTest(className, methodName, instrTest, testCaseDir, mTests);
} else {
Log.d(LOG_TAG, String.format("Creating instrumentation test for %s", mName));
InstrumentationApkTest instrTest = new InstrumentationApkTest();
- return setInstrumentationTest(className, methodName, instrTest, testCaseDir);
+ return setInstrumentationTest(className, methodName, instrTest, testCaseDir, mTests);
}
}
@@ -217,13 +217,15 @@
* @return the populated {@link InstrumentationTest} or <code>null</code>
*/
private InstrumentationTest setInstrumentationTest(String className,
- String methodName, InstrumentationApkTest instrTest, File testCaseDir) {
+ String methodName, InstrumentationApkTest instrTest, File testCaseDir,
+ Collection<TestIdentifier> testsToRun) {
instrTest.setRunName(getUri());
instrTest.setPackageName(mAppNameSpace);
instrTest.setRunnerName(mRunner);
instrTest.setTestPackageName(mTestPackageName);
instrTest.setClassName(className);
instrTest.setMethodName(methodName);
+ instrTest.setTestsToRun(testsToRun, true /* force batch mode */);
// mName means 'apk file name' for instrumentation tests
instrTest.addInstallApk(String.format("%s.apk", mName), mAppNameSpace);
mDigest = generateDigest(testCaseDir, String.format("%s.apk", mName));
