Complete 'continue-session' support
Change the CTS xml generator to read in results from a previous XML file.
The XML generator has also been changed to store results in its own
data structure, rather than reusing CollectingTestListener.
Bug 5171576
Change-Id: Ie4d2aaca47bfb01a7203e9227d4506e2f9a2abd2
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
index 3ae1f06..62827d5 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
@@ -16,6 +16,8 @@
package com.android.cts.tradefed.result;
+import android.tests.getinfo.DeviceInfoConstants;
+
import com.android.cts.tradefed.build.CtsBuildHelper;
import com.android.cts.tradefed.build.CtsBuildProvider;
import com.android.cts.tradefed.device.DeviceInfoCollector;
@@ -27,18 +29,15 @@
import com.android.tradefed.build.IFolderBuildInfo;
import com.android.tradefed.config.Option;
import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.result.CollectingTestListener;
+import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.InputStreamSource;
import com.android.tradefed.result.LogDataType;
-import com.android.tradefed.result.TestResult;
-import com.android.tradefed.result.TestRunResult;
+import com.android.tradefed.result.TestSummary;
import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.StreamUtil;
import org.kxml2.io.KXmlSerializer;
-import android.tests.getinfo.DeviceInfoConstants;
-
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -57,7 +56,7 @@
* <p/>
* Outputs xml in format governed by the cts_result.xsd
*/
-public class CtsXmlResultReporter extends CollectingTestListener {
+public class CtsXmlResultReporter implements ITestInvocationListener {
private static final String LOG_TAG = "CtsXmlResultReporter";
@@ -88,14 +87,18 @@
@Option(name = CtsTest.PLAN_OPTION, description = "the test plan to run.")
private String mPlanName = "NA";
+ // listen in on the continue-session option provided to CtsTest
+ @Option(name = CtsTest.CONTINUE_OPTION, description = "the test result session to continue.")
+ private Integer mContinueSessionId = null;
+
@Option(name = "quiet-output", description = "Mute display of test results.")
private boolean mQuietOutput = false;
protected IBuildInfo mBuildInfo;
-
private String mStartTime;
-
private String mDeviceSerial;
+ private TestResults mResults = new TestResults();
+ private TestPackageResult mCurrentPkgResult = null;
public void setReportDir(File reportDir) {
mReportDir = reportDir;
@@ -106,29 +109,51 @@
*/
@Override
public void invocationStarted(IBuildInfo buildInfo) {
- super.invocationStarted(buildInfo);
- if (mReportDir == null) {
- if (!(buildInfo instanceof IFolderBuildInfo)) {
- throw new IllegalArgumentException("build info is not a IFolderBuildInfo");
- }
-
- IFolderBuildInfo ctsBuild = (IFolderBuildInfo)buildInfo;
- try {
- CtsBuildHelper buildHelper = new CtsBuildHelper(ctsBuild.getRootDir());
- buildHelper.validateStructure();
- mReportDir = buildHelper.getResultsDir();
- } catch (FileNotFoundException e) {
- throw new IllegalArgumentException("Invalid CTS build", e);
- }
+ mBuildInfo = buildInfo;
+ if (!(buildInfo instanceof IFolderBuildInfo)) {
+ throw new IllegalArgumentException("build info is not a IFolderBuildInfo");
}
- // create a unique directory for saving results, using old cts host convention
- // TODO: in future, consider using LogFileSaver to create build-specific directories
- mReportDir = new File(mReportDir, TimeUtil.getResultTimestamp());
- mReportDir.mkdirs();
- mStartTime = getTimestamp();
+ IFolderBuildInfo ctsBuild = (IFolderBuildInfo)buildInfo;
mDeviceSerial = buildInfo.getDeviceSerial() == null ? "unknown_device" :
- buildInfo.getDeviceSerial();
- logResult("Created result dir %s", mReportDir.getName());
+ buildInfo.getDeviceSerial();
+ if (mContinueSessionId != null) {
+ CLog.d("Continuing session %d", mContinueSessionId);
+ // reuse existing directory
+ TestResultRepo resultRepo = new TestResultRepo(getBuildHelper(ctsBuild).getResultsDir());
+ mResults = resultRepo.getResult(mContinueSessionId);
+ if (mResults == null) {
+ throw new IllegalArgumentException(String.format("Could not find session %d",
+ mContinueSessionId));
+ }
+ mPlanName = resultRepo.getSummaries().get(mContinueSessionId).getTestPlan();
+ mStartTime = resultRepo.getSummaries().get(mContinueSessionId).getTimestamp();
+ mReportDir = resultRepo.getReportDir(mContinueSessionId);
+ } else {
+ if (mReportDir == null) {
+ mReportDir = getBuildHelper(ctsBuild).getResultsDir();
+ }
+ // create a unique directory for saving results, using old cts host convention
+ // TODO: in future, consider using LogFileSaver to create build-specific directories
+ mReportDir = new File(mReportDir, TimeUtil.getResultTimestamp());
+ mReportDir.mkdirs();
+ mStartTime = getTimestamp();
+ logResult("Created result dir %s", mReportDir.getName());
+ }
+ }
+
+ /**
+ * Helper method to retrieve the {@link CtsBuildHelper}.
+ * @param ctsBuild
+ * @return
+ */
+ CtsBuildHelper getBuildHelper(IFolderBuildInfo ctsBuild) {
+ CtsBuildHelper buildHelper = new CtsBuildHelper(ctsBuild.getRootDir());
+ try {
+ buildHelper.validateStructure();
+ } catch (FileNotFoundException e) {
+ throw new IllegalArgumentException("Invalid CTS build", e);
+ }
+ return buildHelper;
}
/**
@@ -149,30 +174,51 @@
}
}
+ /**
+ * {@inheritDoc}
+ */
@Override
public void testRunStarted(String name, int numTests) {
+ if (mCurrentPkgResult != null && !name.equals(mCurrentPkgResult.getAppPackageName())) {
+ // display results from previous run
+ logCompleteRun(mCurrentPkgResult);
+ }
if (name.equals(DeviceInfoCollector.APP_PACKAGE_NAME)) {
logResult("Collecting device info");
- } else if (!name.equals(getCurrentRunResults().getName())) {
- // this is a new run
- if (getCurrentRunResults().isRunComplete()) {
- // display results from previous run
- logCompleteRun(getCurrentRunResults());
- }
+ } else if (mCurrentPkgResult == null || !name.equals(
+ mCurrentPkgResult.getAppPackageName())) {
logResult("-----------------------------------------");
logResult("Test package %s started", name);
logResult("-----------------------------------------");
}
- super.testRunStarted(name, numTests);
+ mCurrentPkgResult = mResults.getOrCreatePackage(name);
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testStarted(TestIdentifier test) {
+ mCurrentPkgResult.insertTest(test);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testFailed(TestFailure status, TestIdentifier test, String trace) {
+ mCurrentPkgResult.reportTestFailure(test, CtsTestStatus.FAIL, trace);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
@Override
public void testEnded(TestIdentifier test, Map<String, String> testMetrics) {
- super.testEnded(test, testMetrics);
- TestRunResult results = getCurrentRunResults();
- TestResult result = results.getTestResults().get(test);
+ mCurrentPkgResult.reportTestEnded(test);
+ Test result = mCurrentPkgResult.findTest(test);
String stack = result.getStackTrace() == null ? "" : "\n" + result.getStackTrace();
- logResult("%s#%s %s %s", test.getClassName(), test.getTestName(), result.getStatus(),
+ logResult("%s#%s %s %s", test.getClassName(), test.getTestName(), result.getResult(),
stack);
}
@@ -180,12 +226,19 @@
* {@inheritDoc}
*/
@Override
+ public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) {
+ mCurrentPkgResult.populateMetrics(runMetrics);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public void invocationEnded(long elapsedTime) {
// display the results of the last completed run
- if (getCurrentRunResults().isRunComplete()) {
- logCompleteRun(getCurrentRunResults());
+ if (mCurrentPkgResult != null) {
+ logCompleteRun(mCurrentPkgResult);
}
- super.invocationEnded(elapsedTime);
createXmlResult(mReportDir, mStartTime, elapsedTime);
copyFormattingFiles(mReportDir);
zipResults(mReportDir);
@@ -199,16 +252,15 @@
}
}
- private void logCompleteRun(TestRunResult runResults) {
- if (runResults.getName().equals(DeviceInfoCollector.APP_PACKAGE_NAME)) {
+ private void logCompleteRun(TestPackageResult pkgResult) {
+ if (pkgResult.getAppPackageName().equals(DeviceInfoCollector.APP_PACKAGE_NAME)) {
logResult("Device info collection complete");
return;
}
logResult("%s package complete: Passed %d, Failed %d, Not Executed %d",
- runResults.getName(), runResults.getNumPassedTests(),
- runResults.getNumFailedTests() +
- runResults.getNumErrorTests(),
- runResults.getNumIncompleteTests());
+ pkgResult.getAppPackageName(), pkgResult.countTests(CtsTestStatus.PASS),
+ pkgResult.countTests(CtsTestStatus.FAIL),
+ pkgResult.countTests(CtsTestStatus.NOT_EXECUTED));
}
/**
@@ -230,8 +282,10 @@
serializeResultsDoc(serializer, startTimestamp, endTime);
serializer.endDocument();
String msg = String.format("XML test result file generated at %s. Passed %d, " +
- "Failed %d, Not Executed %d", mReportDir.getName(), getNumPassedTests(),
- getNumFailedTests() + getNumErrorTests(), getNumIncompleteTests());
+ "Failed %d, Not Executed %d", mReportDir.getName(),
+ mResults.countTests(CtsTestStatus.PASS),
+ mResults.countTests(CtsTestStatus.FAIL),
+ mResults.countTests(CtsTestStatus.NOT_EXECUTED));
logResult(msg);
logResult("Time: %s", TimeUtil.formatElapsedTime(elapsedTime));
} catch (IOException e) {
@@ -260,7 +314,7 @@
serializeDeviceInfo(serializer);
serializeHostInfo(serializer);
serializeTestSummary(serializer);
- serializeTestResults(serializer);
+ mResults.serialize(serializer);
// TODO: not sure why, but the serializer doesn't like this statement
//serializer.endTag(ns, RESULT_TAG);
}
@@ -273,15 +327,16 @@
private void serializeDeviceInfo(KXmlSerializer serializer) throws IOException {
serializer.startTag(ns, "DeviceInfo");
- TestRunResult deviceInfoResult = findRunResult(DeviceInfoCollector.APP_PACKAGE_NAME);
- if (deviceInfoResult == null) {
- Log.w(LOG_TAG, String.format("Could not find device info run %s",
- DeviceInfoCollector.APP_PACKAGE_NAME));
+ Map<String, String> deviceInfoMetrics = mResults.getDeviceInfoMetrics();
+ if (deviceInfoMetrics == null || deviceInfoMetrics.isEmpty()) {
+ // this might be expected, if device info collection was turned off
+ CLog.d("Could not find device info");
return;
}
+
// Extract metrics that need extra handling, and then dump the remainder into BuildInfo
Map<String, String> metricsCopy = new HashMap<String, String>(
- deviceInfoResult.getRunMetrics());
+ deviceInfoMetrics);
serializer.startTag(ns, "Screen");
String screenWidth = metricsCopy.remove(DeviceInfoConstants.SCREEN_WIDTH);
String screenHeight = metricsCopy.remove(DeviceInfoConstants.SCREEN_HEIGHT);
@@ -390,21 +445,6 @@
}
/**
- * Finds the {@link TestRunResult} with the given name.
- *
- * @param runName
- * @return the {@link TestRunResult}
- */
- private TestRunResult findRunResult(String runName) {
- for (TestRunResult runResult : getRunResults()) {
- if (runResult.getName().equals(runName)) {
- return runResult;
- }
- }
- return null;
- }
-
- /**
* Output the host info XML.
*
* @param serializer
@@ -451,69 +491,18 @@
*/
private void serializeTestSummary(KXmlSerializer serializer) throws IOException {
serializer.startTag(ns, SUMMARY_TAG);
- serializer.attribute(ns, FAILED_ATTR, Integer.toString(getNumErrorTests() +
- getNumFailedTests()));
- serializer.attribute(ns, NOT_EXECUTED_ATTR, Integer.toString(getNumIncompleteTests()));
+ serializer.attribute(ns, FAILED_ATTR, Integer.toString(mResults.countTests(
+ CtsTestStatus.FAIL)));
+ serializer.attribute(ns, NOT_EXECUTED_ATTR, Integer.toString(mResults.countTests(
+ CtsTestStatus.NOT_EXECUTED)));
// ignore timeouts - these are reported as errors
serializer.attribute(ns, TIMEOUT_ATTR, "0");
- serializer.attribute(ns, PASS_ATTR, Integer.toString(getNumPassedTests()));
+ serializer.attribute(ns, PASS_ATTR, Integer.toString(mResults.countTests(
+ CtsTestStatus.PASS)));
serializer.endTag(ns, SUMMARY_TAG);
}
/**
- * Output the detailed test results XML.
- *
- * @param serializer
- * @throws IOException
- */
- private void serializeTestResults(KXmlSerializer serializer) throws IOException {
- for (TestRunResult runResult : getRunResults()) {
- serializeTestRunResult(serializer, runResult);
- }
- }
-
- /**
- * Output the XML for one test run aka test package.
- *
- * @param serializer
- * @param runResult the {@link TestRunResult}
- * @throws IOException
- */
- private void serializeTestRunResult(KXmlSerializer serializer, TestRunResult runResult)
- throws IOException {
- if (runResult.getName().equals(DeviceInfoCollector.APP_PACKAGE_NAME)) {
- // ignore run results for the info collecting packages
- return;
- }
- TestPackageResult packageResult = new TestPackageResult();
- packageResult.setName(getMetric(runResult, CtsTest.PACKAGE_NAME_METRIC));
- packageResult.setAppPackageName(runResult.getName());
- packageResult.setDigest(getMetric(runResult, CtsTest.PACKAGE_DIGEST_METRIC));
- // organize the tests into data structures that mirror the expected xml output.
- for (Map.Entry<TestIdentifier, TestResult> testEntry : runResult.getTestResults()
- .entrySet()) {
- packageResult.insertTest(testEntry.getKey(), testEntry.getValue());
- }
- // dump the results
- packageResult.serialize(serializer);
- }
-
- /**
- * Helper method to retrieve the metric value with given name, or blank if not found
- *
- * @param runResult
- * @param string
- * @return
- */
- private String getMetric(TestRunResult runResult, String keyName) {
- String value = runResult.getRunMetrics().get(keyName);
- if (value == null) {
- return "";
- }
- return value;
- }
-
- /**
* Creates the output stream to use for test results. Exposed for mocking.
* @param mReportPath
*/
@@ -570,4 +559,36 @@
String getTimestamp() {
return TimeUtil.getTimestamp();
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testRunFailed(String errorMessage) {
+ // ignore
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testRunStopped(long elapsedTime) {
+ // ignore
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void invocationFailed(Throwable cause) {
+ // ignore
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public TestSummary getSummary() {
+ return null;
+ }
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/ITestResultRepo.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/ITestResultRepo.java
index bed2719..d672f41 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/ITestResultRepo.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/ITestResultRepo.java
@@ -15,6 +15,7 @@
*/
package com.android.cts.tradefed.result;
+import java.io.File;
import java.util.List;
/**
@@ -37,4 +38,11 @@
*/
public TestResults getResult(int sessionId);
+ /**
+ * Get the report directory for given result
+ * @param sessionId
+ * @return
+ */
+ public File getReportDir(int sessionId);
+
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java
index 184b8ab..917ccbe 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java
@@ -15,9 +15,7 @@
*/
package com.android.cts.tradefed.result;
-import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.TestResult;
-import com.android.tradefed.result.TestResult.TestStatus;
import org.kxml2.io.KXmlSerializer;
import org.xmlpull.v1.XmlPullParser;
@@ -40,7 +38,7 @@
private static final String STACK_TAG = "StackTrace";
private String mName;
- private String mResult;
+ private CtsTestStatus mResult;
private String mStartTime;
private String mEndTime;
private String mMessage;
@@ -56,18 +54,12 @@
* Create a {@link Test} from a {@link TestResult}.
*
* @param name
- * @param result
*/
- public Test(String name, TestResult result) {
+ public Test(String name) {
mName = name;
- mResult = convertStatus(result.getStatus());
- mStartTime = TimeUtil.getTimestamp(result.getStartTime());
- mEndTime = TimeUtil.getTimestamp(result.getEndTime());
- if (result.getStackTrace() != null) {
- String sanitizedStack = sanitizeStackTrace(result.getStackTrace());
- mMessage = getFailureMessageFromStackTrace(sanitizedStack);
- mStackTrace = sanitizedStack;
- }
+ mResult = CtsTestStatus.NOT_EXECUTED;
+ mStartTime = TimeUtil.getTimestamp();
+ updateEndTime();
}
/**
@@ -84,7 +76,7 @@
return mName;
}
- public String getResult() {
+ public CtsTestStatus getResult() {
return mResult;
}
@@ -104,6 +96,20 @@
return mStackTrace;
}
+ public void setStackTrace(String stackTrace) {
+
+ mStackTrace = sanitizeStackTrace(stackTrace);
+ mMessage = getFailureMessageFromStackTrace(mStackTrace);
+ }
+
+ public void updateEndTime() {
+ mEndTime = TimeUtil.getTimestamp();
+ }
+
+ public void setResultStatus(CtsTestStatus status) {
+ mResult = status;
+ }
+
/**
* Serialize this object and all its contents to XML.
*
@@ -114,7 +120,7 @@
throws IOException {
serializer.startTag(CtsXmlResultReporter.ns, TAG);
serializer.attribute(CtsXmlResultReporter.ns, NAME_ATTR, getName());
- serializer.attribute(CtsXmlResultReporter.ns, RESULT_ATTR, mResult);
+ serializer.attribute(CtsXmlResultReporter.ns, RESULT_ATTR, mResult.getValue());
serializer.attribute(CtsXmlResultReporter.ns, STARTTIME_ATTR, mStartTime);
serializer.attribute(CtsXmlResultReporter.ns, ENDTIME_ATTR, mEndTime);
@@ -132,27 +138,6 @@
}
/**
- * Convert a {@link TestStatus} to the result text to output in XML
- *
- * @param status the {@link TestStatus}
- * @return
- */
- private String convertStatus(TestStatus status) {
- switch (status) {
- case ERROR:
- return CtsTestStatus.FAIL.getValue();
- case FAILURE:
- return CtsTestStatus.FAIL.getValue();
- case PASSED:
- return CtsTestStatus.PASS.getValue();
- case INCOMPLETE:
- return CtsTestStatus.NOT_EXECUTED.getValue();
- }
- CLog.w("Unrecognized status %s", status);
- return CtsTestStatus.FAIL.getValue();
- }
-
- /**
* Strip out any invalid XML characters that might cause the report to be unviewable.
* http://www.w3.org/TR/REC-xml/#dt-character
*/
@@ -187,7 +172,7 @@
"invalid XML: Expected %s tag but received %s", TAG, parser.getName()));
}
setName(getAttribute(parser, NAME_ATTR));
- mResult = getAttribute(parser, RESULT_ATTR);
+ mResult = CtsTestStatus.getStatus(getAttribute(parser, RESULT_ATTR));
mStartTime = getAttribute(parser, STARTTIME_ATTR);
mEndTime = getAttribute(parser, ENDTIME_ATTR);
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestCase.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestCase.java
index 138fe7c..fdb8b3b 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestCase.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestCase.java
@@ -16,7 +16,6 @@
package com.android.cts.tradefed.result;
import com.android.ddmlib.testrunner.TestIdentifier;
-import com.android.tradefed.result.TestResult;
import com.android.tradefed.util.ArrayUtil;
import org.kxml2.io.KXmlSerializer;
@@ -67,24 +66,17 @@
}
/**
- * Inserts given test result
- *
* @param testName
- * @param testResult
+ * @param insertIfMissing
+ * @return
*/
- public void insertTest(String testName, TestResult testResult) {
- Test t = new Test(testName, testResult);
- insertTest(t);
- }
-
- /**
- * Inserts given test result
- *
- * @param testName
- * @param testResult
- */
- private void insertTest(Test test) {
- mChildTestMap.put(test.getName(), test);
+ public Test findTest(String testName, boolean insertIfMissing) {
+ Test t = mChildTestMap.get(testName);
+ if (t == null && insertIfMissing) {
+ t = new Test(testName);
+ mChildTestMap.put(t.getName(), t);
+ }
+ return t;
}
/**
@@ -122,7 +114,7 @@
if (eventType == XmlPullParser.START_TAG && parser.getName().equals(Test.TAG)) {
Test test = new Test();
test.parse(parser);
- insertTest(test);
+ mChildTestMap.put(test.getName(), test);
} else if (eventType == XmlPullParser.END_TAG && parser.getName().equals(TAG)) {
return;
}
@@ -145,7 +137,7 @@
}
String fullClassName = ArrayUtil.join(".", parentSuiteNames);
for (Test test : mChildTestMap.values()) {
- if (resultFilter.getValue().equals(test.getResult())) {
+ if (resultFilter.equals(test.getResult())) {
tests.add(new TestIdentifier(fullClassName, test.getName()));
}
}
@@ -153,4 +145,20 @@
parentSuiteNames.removeLast();
}
}
+
+ /**
+ * Count the number of tests in this {@link TestCase} with given status.
+ *
+ * @param status
+ * @return the test count
+ */
+ public int countTests(CtsTestStatus status) {
+ int total = 0;
+ for (Test test : mChildTestMap.values()) {
+ if (test.getResult().equals(status)) {
+ total++;
+ }
+ }
+ return total;
+ }
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestPackageResult.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestPackageResult.java
index 8480803..7d5db75 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestPackageResult.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestPackageResult.java
@@ -15,9 +15,9 @@
*/
package com.android.cts.tradefed.result;
+import com.android.cts.tradefed.testtype.CtsTest;
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.result.TestResult;
import org.kxml2.io.KXmlSerializer;
import org.xmlpull.v1.XmlPullParser;
@@ -27,8 +27,10 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
/**
* Data structure for a CTS test package result.
@@ -48,6 +50,8 @@
private String mName;
private String mDigest;
+ private Map<String, String> mMetrics = new HashMap<String, String>();
+
private TestSuite mSuiteRoot = new TestSuite(null);
public void setAppPackageName(String appPackageName) {
@@ -87,17 +91,31 @@
* @param testId
* @param testResult
*/
- public void insertTest(TestIdentifier testId, TestResult testResult) {
+ public Test insertTest(TestIdentifier testId) {
+ return findTest(testId, true);
+ }
+
+ private Test findTest(TestIdentifier testId, boolean insertIfMissing) {
List<String> classNameSegments = new LinkedList<String>();
Collections.addAll(classNameSegments, testId.getClassName().split("\\."));
if (classNameSegments.size() <= 0) {
CLog.e("Unrecognized package name format for test class '%s'",
testId.getClassName());
- } else {
- String testCaseName = classNameSegments.remove(classNameSegments.size()-1);
- mSuiteRoot.insertTest(classNameSegments, testCaseName, testId.getTestName(),
- testResult);
+ // should never happen
+ classNameSegments.add("UnknownTestClass");
}
+ String testCaseName = classNameSegments.remove(classNameSegments.size()-1);
+ return mSuiteRoot.findTest(classNameSegments, testCaseName, testId.getTestName(), insertIfMissing);
+ }
+
+
+ /**
+ * Find the test result for given {@link TestIdentifier}.
+ * @param testId
+ * @return the {@link Test} or <code>null</code>
+ */
+ public Test findTest(TestIdentifier testId) {
+ return findTest(testId, false);
}
/**
@@ -108,10 +126,10 @@
*/
public void serialize(KXmlSerializer serializer) throws IOException {
serializer.startTag(ns, TAG);
- serializer.attribute(ns, NAME_ATTR, mName);
- serializer.attribute(ns, APP_PACKAGE_NAME_ATTR, mAppPackageName);
- serializer.attribute(ns, DIGEST_ATTR, getDigest());
- if (mName.equals(SIGNATURE_TEST_PKG)) {
+ serializeAttribute(serializer, NAME_ATTR, mName);
+ serializeAttribute(serializer, APP_PACKAGE_NAME_ATTR, mAppPackageName);
+ serializeAttribute(serializer, DIGEST_ATTR, getDigest());
+ if (SIGNATURE_TEST_PKG.equals(mName)) {
serializer.attribute(ns, "signatureCheck", "true");
}
mSuiteRoot.serialize(serializer);
@@ -119,6 +137,21 @@
}
/**
+ * Helper method to serialize attributes.
+ * Can handle null values. Useful for cases where test package has not been fully populated
+ * such as when unit testing.
+ *
+ * @param attrName
+ * @param attrValue
+ * @throws IOException
+ */
+ private void serializeAttribute(KXmlSerializer serializer, String attrName, String attrValue)
+ throws IOException {
+ attrValue = attrValue == null ? "" : attrValue;
+ serializer.attribute(ns, attrName, attrValue);
+ }
+
+ /**
* Populates this class with package result data parsed from XML.
*
* @param parser the {@link XmlPullParser}. Expected to be pointing at start
@@ -159,4 +192,63 @@
mSuiteRoot.addTestsWithStatus(tests, suiteNames, resultFilter);
return tests;
}
+
+ /**
+ * Populate values in this package result from run metrics
+ * @param runResult
+ */
+ public void populateMetrics(Map<String, String> metrics) {
+ String name = metrics.get(CtsTest.PACKAGE_NAME_METRIC);
+ if (name != null) {
+ setName(name);
+ }
+ String digest = metrics.get(CtsTest.PACKAGE_DIGEST_METRIC);
+ if (digest != null) {
+ setDigest(digest);
+ }
+ mMetrics.putAll(metrics);
+ }
+
+ /**
+ * Report the given test as a failure.
+ *
+ * @param test
+ * @param status
+ * @param trace
+ */
+ public void reportTestFailure(TestIdentifier test, CtsTestStatus status, String trace) {
+ Test result = findTest(test);
+ result.setResultStatus(status);
+ result.setStackTrace(trace);
+ }
+
+ /**
+ * Report that the given test has completed.
+ *
+ * @param test
+ */
+ public void reportTestEnded(TestIdentifier test) {
+ Test result = findTest(test);
+ if (!result.getResult().equals(CtsTestStatus.FAIL)) {
+ result.setResultStatus(CtsTestStatus.PASS);
+ }
+ result.updateEndTime();
+ }
+
+ /**
+ * Return the number of tests with given status
+ *
+ * @param status
+ * @return the total number of tests with given status
+ */
+ public int countTests(CtsTestStatus status) {
+ return mSuiteRoot.countTests(status);
+ }
+
+ /**
+ * @return
+ */
+ public Map<String, String> getMetrics() {
+ return mMetrics;
+ }
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestResultRepo.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestResultRepo.java
index 9178e91..fd42892 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestResultRepo.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestResultRepo.java
@@ -54,12 +54,17 @@
File resultFile = new File(resultList.get(i),
CtsXmlResultReporter.TEST_RESULT_FILE_NAME);
if (resultFile.exists()) {
- mResultDirs.add(i, resultList.get(i));
+ mResultDirs.add(resultList.get(i));
}
}
}
}
+ @Override
+ public File getReportDir(int sessionId) {
+ return mResultDirs.get(sessionId);
+ }
+
private ITestSummary parseSummary(int id, File resultDir) {
TestSummaryXml result = new TestSummaryXml(id, resultDir.getName());
try {
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestResults.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestResults.java
index 6900e58..a874227 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestResults.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestResults.java
@@ -15,12 +15,21 @@
*/
package com.android.cts.tradefed.result;
+import com.android.cts.tradefed.device.DeviceInfoCollector;
+import com.android.tradefed.log.LogUtil.CLog;
+
+import org.kxml2.io.KXmlSerializer;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
/**
* Data structure for the detailed CTS test results.
@@ -29,7 +38,8 @@
*/
class TestResults extends AbstractXmlPullParser {
- private List<TestPackageResult> mPackages = new ArrayList<TestPackageResult>();
+ private Map<String, TestPackageResult> mPackageMap = new LinkedHashMap<String, TestPackageResult>();
+ private TestPackageResult mDeviceInfoPkg = new TestPackageResult();
/**
* {@inheritDoc}
@@ -42,16 +52,91 @@
TestPackageResult.TAG)) {
TestPackageResult pkg = new TestPackageResult();
pkg.parse(parser);
- mPackages.add(pkg);
+ if (pkg.getAppPackageName() != null) {
+ mPackageMap.put(pkg.getAppPackageName(), pkg);
+ } else {
+ CLog.w("Found package with no app package name");
+ }
}
eventType = parser.next();
}
}
/**
- * @return the list of parsed {@link TestPackageResult}.
+ * @return the list of {@link TestPackageResult}.
*/
- public List<TestPackageResult> getPackages() {
- return mPackages;
+ public Collection<TestPackageResult> getPackages() {
+ return mPackageMap.values();
+ }
+
+ /**
+ * Count the number of tests with given status
+ * @param pass
+ * @return
+ */
+ public int countTests(CtsTestStatus status) {
+ int total = 0;
+ for (TestPackageResult result : mPackageMap.values()) {
+ total += result.countTests(status);
+ }
+ return total;
+ }
+
+ /**
+ * @return
+ */
+ public Map<String, String> getDeviceInfoMetrics() {
+ return mDeviceInfoPkg.getMetrics();
+ }
+
+ /**
+ * @param mCurrentPkgResult
+ */
+ public void addPackageResult(TestPackageResult pkgResult) {
+ mPackageMap.put(pkgResult.getName(), pkgResult);
+ }
+
+ /**
+ * @param serializer
+ * @throws IOException
+ */
+ public void serialize(KXmlSerializer serializer) throws IOException {
+ // sort before serializing
+ List<TestPackageResult> pkgs = new ArrayList<TestPackageResult>(mPackageMap.values());
+ Collections.sort(pkgs, new PkgComparator());
+ for (TestPackageResult r : pkgs) {
+ r.serialize(serializer);
+ }
+ }
+
+ private static class PkgComparator implements Comparator<TestPackageResult> {
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int compare(TestPackageResult o1, TestPackageResult o2) {
+ return o1.getAppPackageName().compareTo(o2.getAppPackageName());
+ }
+
+ }
+
+ /**
+ * Return existing package with given app package name. If not found, create a new one.
+ * @param name
+ * @return
+ */
+ public TestPackageResult getOrCreatePackage(String appPackageName) {
+ if (appPackageName.equals(DeviceInfoCollector.APP_PACKAGE_NAME)) {
+ mDeviceInfoPkg.setAppPackageName(DeviceInfoCollector.APP_PACKAGE_NAME);
+ return mDeviceInfoPkg ;
+ }
+ TestPackageResult pkgResult = mPackageMap.get(appPackageName);
+ if (pkgResult == null) {
+ pkgResult = new TestPackageResult();
+ pkgResult.setAppPackageName(appPackageName);
+ mPackageMap.put(appPackageName, pkgResult);
+ }
+ return pkgResult;
}
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestSuite.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestSuite.java
index 426ff2d..df1dceb 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestSuite.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestSuite.java
@@ -74,16 +74,16 @@
* @param testName the test method name
* @param testResult the {@link TestResult}
*/
- public void insertTest(List<String> suiteNames, String testClassName, String testName,
- TestResult testResult) {
+ public Test findTest(List<String> suiteNames, String testClassName, String testName,
+ boolean insertIfMissing) {
if (suiteNames.size() <= 0) {
// no more package segments
TestCase testCase = getTestCase(testClassName);
- testCase.insertTest(testName, testResult);
+ return testCase.findTest(testName, insertIfMissing);
} else {
String rootName = suiteNames.remove(0);
TestSuite suite = getTestSuite(rootName);
- suite.insertTest(suiteNames, testClassName, testName, testResult);
+ return suite.findTest(suiteNames, testClassName, testName, insertIfMissing);
}
}
@@ -221,4 +221,21 @@
parentSuiteNames.removeLast();
}
}
+
+ /**
+ * Count the number of tests in this {@link TestSuite} with given status.
+ *
+ * @param status
+ * @return the test count
+ */
+ public int countTests(CtsTestStatus status) {
+ int total = 0;
+ for (TestSuite suite : mChildSuiteMap.values()) {
+ total += suite.countTests(status);
+ }
+ for (TestCase testCase : mChildTestCaseMap.values()) {
+ total += testCase.countTests(status);
+ }
+ return total;
+ }
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java
index 68bb62e..e8fdad81 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/testtype/ResultFilter.java
@@ -130,14 +130,16 @@
*/
public void reportUnexecutedTests() {
for (Map.Entry<String, Collection<TestIdentifier>> entry : mRemainingTestsMap.entrySet()) {
- super.testRunStarted(entry.getKey(), entry.getValue().size());
- for (TestIdentifier test : entry.getValue()) {
- // an unexecuted test is currently reported as a 'testStarted' event without a
- // 'testEnded'. TODO: consider adding an explict API for reporting an unexecuted
- // test
- super.testStarted(test);
+ if (!entry.getValue().isEmpty()) {
+ super.testRunStarted(entry.getKey(), entry.getValue().size());
+ for (TestIdentifier test : entry.getValue()) {
+ // an unexecuted test is currently reported as a 'testStarted' event without a
+ // 'testEnded'. TODO: consider adding an explict API for reporting an unexecuted
+ // test
+ super.testStarted(test);
+ }
+ super.testRunEnded(0, new HashMap<String,String>());
}
- super.testRunEnded(0, new HashMap<String,String>());
}
}
}
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java
index 4e6b914..7566fce 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java
@@ -15,13 +15,17 @@
*/
package com.android.cts.tradefed.result;
-import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.ddmlib.testrunner.ITestRunListener.TestFailure;
-import com.android.tradefed.build.BuildInfo;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.build.IFolderBuildInfo;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.XmlResultReporter;
import com.android.tradefed.util.FileUtil;
+import junit.framework.TestCase;
+
+import org.easymock.EasyMock;
+
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
@@ -29,8 +33,6 @@
import java.util.Collections;
import java.util.Map;
-import junit.framework.TestCase;
-
/**
* Unit tests for {@link XmlResultReporter}.
*/
@@ -39,6 +41,7 @@
private CtsXmlResultReporter mResultReporter;
private ByteArrayOutputStream mOutputStream;
private File mReportDir;
+ private IFolderBuildInfo mMockBuild;
/**
* {@inheritDoc}
@@ -62,6 +65,7 @@
// TODO: use mock file dir instead
mReportDir = FileUtil.createTempDir("foo");
mResultReporter.setReportDir(mReportDir);
+ mMockBuild = EasyMock.createNiceMock(IFolderBuildInfo.class);
}
@Override
@@ -83,7 +87,7 @@
final String expectedSummaryOutput =
"<Summary failed=\"0\" notExecuted=\"0\" timeout=\"0\" pass=\"0\" />";
final String expectedEndTag = "</TestResult>";
- mResultReporter.invocationStarted(new BuildInfo("1", "test", "test"));
+ mResultReporter.invocationStarted(mMockBuild);
mResultReporter.invocationEnded(1);
String actualOutput = getOutput();
assertTrue(actualOutput.startsWith(expectedHeaderOutput));
@@ -101,7 +105,7 @@
public void testSinglePass() {
Map<String, String> emptyMap = Collections.emptyMap();
final TestIdentifier testId = new TestIdentifier("com.foo.FooTest", "testFoo");
- mResultReporter.invocationStarted(new BuildInfo());
+ mResultReporter.invocationStarted(mMockBuild);
mResultReporter.testRunStarted("run", 1);
mResultReporter.testStarted(testId);
mResultReporter.testEnded(testId, emptyMap);
@@ -128,7 +132,7 @@
Map<String, String> emptyMap = Collections.emptyMap();
final TestIdentifier testId = new TestIdentifier("FooTest", "testFoo");
final String trace = "this is a trace\nmore trace";
- mResultReporter.invocationStarted(new BuildInfo());
+ mResultReporter.invocationStarted(mMockBuild);
mResultReporter.testRunStarted("run", 1);
mResultReporter.testStarted(testId);
mResultReporter.testFailed(TestFailure.FAILURE, testId, trace);
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestPackageResultTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestPackageResultTest.java
index b3c903b..df80dbb 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestPackageResultTest.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestPackageResultTest.java
@@ -16,8 +16,6 @@
package com.android.cts.tradefed.result;
import com.android.ddmlib.testrunner.TestIdentifier;
-import com.android.tradefed.result.TestResult;
-import com.android.tradefed.result.TestResult.TestStatus;
import junit.framework.TestCase;
@@ -34,14 +32,11 @@
public void testGetTestsWithStatus() {
TestPackageResult pkgResult = new TestPackageResult();
TestIdentifier excludedTest = new TestIdentifier("com.example.ExampleTest", "testPass");
- TestResult passed = new TestResult();
- passed.setStatus(TestStatus.PASSED);
- pkgResult.insertTest(excludedTest, passed);
+ pkgResult.insertTest(excludedTest);
+ pkgResult.reportTestEnded(excludedTest);
TestIdentifier includedTest = new TestIdentifier("com.example.ExampleTest",
"testNotExecuted");
- TestResult notExecuted = new TestResult();
- notExecuted.setStatus(TestStatus.INCOMPLETE);
- pkgResult.insertTest(includedTest, notExecuted);
+ pkgResult.insertTest(includedTest);
Collection<TestIdentifier> tests = pkgResult.getTestsWithStatus(
CtsTestStatus.NOT_EXECUTED);
assertEquals(1, tests.size());
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestResultsTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestResultsTest.java
index a0360cd..75f545e 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestResultsTest.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestResultsTest.java
@@ -64,7 +64,7 @@
TestResults parser = new TestResults();
parser.parse(new StringReader(TEST_PACKAGE_FULL));
assertEquals(1, parser.getPackages().size());
- TestPackageResult pkg = parser.getPackages().get(0);
+ TestPackageResult pkg = parser.getPackages().iterator().next();
assertEquals("pkgName", pkg.getName());
assertEquals("appPkgName", pkg.getAppPackageName());
assertEquals("digValue", pkg.getDigest());
@@ -91,7 +91,7 @@
TestResults parser = new TestResults();
parser.parse(new StringReader(TEST_FULL));
assertEquals(1, parser.getPackages().size());
- TestPackageResult pkg = parser.getPackages().get(0);
+ TestPackageResult pkg = parser.getPackages().iterator().next();
TestSuite comSuite = pkg.getTestSuites().iterator().next();
assertEquals("com", comSuite.getName());
TestSuite exampleSuite = comSuite.getTestSuites().iterator().next();
