Add run history information to XML test report.
This is to collect OEM's full CTS execution time by recording the run
time of each retry including initial CTS run in test report.
Bug: 135619175
Test: cts-tradefed run cts-unit-tests
Change-Id: Id5f83f5c1ca083639b9d906fb85c0fbce645d593
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
index 32eff01..2267720 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
@@ -27,6 +27,7 @@
import com.android.compatibility.common.util.IModuleResult;
import com.android.compatibility.common.util.ITestResult;
import com.android.compatibility.common.util.InvocationResult;
+import com.android.compatibility.common.util.InvocationResult.RunHistory;
import com.android.compatibility.common.util.MetricsStore;
import com.android.compatibility.common.util.ReportLog;
import com.android.compatibility.common.util.ResultHandler;
@@ -64,6 +65,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.xml.XmlEscapers;
+import com.google.gson.Gson;
import org.xmlpull.v1.XmlPullParserException;
@@ -75,6 +77,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -97,6 +100,9 @@
private static final String CTS_PREFIX = "cts:";
private static final String BUILD_INFO = CTS_PREFIX + "build_";
private static final String LATEST_LINK_NAME = "latest";
+ /** Used to get run history from the test result of last run. */
+ private static final String RUN_HISTORY_KEY = "run_history";
+
public static final String BUILD_BRAND = "build_brand";
public static final String BUILD_DEVICE = "build_device";
@@ -579,6 +585,22 @@
return;
}
+ // Get run history from the test result of last run and add the run history of the current
+ // run to it.
+ // TODO(b/137973382): avoid casting by move the method to interface level.
+ Collection<RunHistory> runHistories = ((InvocationResult) mResult).getRunHistories();
+ String runHistoryJSON = mResult.getInvocationInfo().get(RUN_HISTORY_KEY);
+ Gson gson = new Gson();
+ if (runHistoryJSON != null) {
+ RunHistory[] runHistoryArray = gson.fromJson(runHistoryJSON, RunHistory[].class);
+ Collections.addAll(runHistories, runHistoryArray);
+ }
+ RunHistory newRun = new RunHistory();
+ newRun.startTime = mResult.getStartTime();
+ newRun.endTime = newRun.startTime + mElapsedTime;
+ runHistories.add(newRun);
+ mResult.addInvocationInfo(RUN_HISTORY_KEY, gson.toJson(runHistories));
+
try {
// Zip the full test results directory.
copyDynamicConfigFiles();
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
index a179978..82c4d8f 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
@@ -549,4 +549,75 @@
TestStatus.PASS,
result1.getResultStatus());
}
+
+ /** Ensure that the run history of the current run is added to previous run history. */
+ public void testRetryWithRunHistory() throws Exception {
+ mReporter.invocationStarted(mContext);
+
+ // Set up IInvocationResult with existing results from previous session
+ mReporter.testRunStarted(ID, 2);
+ IInvocationResult invocationResult = mReporter.getResult();
+ IModuleResult moduleResult = invocationResult.getOrCreateModule(ID);
+ ICaseResult caseResult = moduleResult.getOrCreateResult(CLASS);
+ ITestResult testResult1 = caseResult.getOrCreateResult(METHOD_1);
+ testResult1.setResultStatus(TestStatus.PASS);
+ testResult1.setRetry(true);
+ ITestResult testResult2 = caseResult.getOrCreateResult(METHOD_2);
+ testResult2.setResultStatus(TestStatus.FAIL);
+ testResult2.setStackTrace(STACK_TRACE);
+ testResult2.setRetry(true);
+ // Set up IInvocationResult with the run history of previous runs.
+ invocationResult.addInvocationInfo(
+ "run_history", "[{\"startTime\":1,\"endTime\":2},{\"startTime\":3,\"endTime\":4}]");
+
+ // Flip results for the current session
+ TestDescription test1 = new TestDescription(CLASS, METHOD_1);
+ mReporter.testStarted(test1);
+ mReporter.testFailed(test1, STACK_TRACE);
+ mReporter.testEnded(test1, new HashMap<String, Metric>());
+ TestDescription test2 = new TestDescription(CLASS, METHOD_2);
+ mReporter.testStarted(test2);
+ mReporter.testEnded(test2, new HashMap<String, Metric>());
+
+ mReporter.testRunEnded(10, new HashMap<String, Metric>());
+ mReporter.invocationEnded(10);
+
+ // Verification that results have been overwritten.
+ IInvocationResult result = mReporter.getResult();
+ assertEquals("Expected 1 pass", 1, result.countResults(TestStatus.PASS));
+ assertEquals("Expected 1 failure", 1, result.countResults(TestStatus.FAIL));
+ List<IModuleResult> modules = result.getModules();
+ assertEquals("Expected 1 module", 1, modules.size());
+ IModuleResult module = modules.get(0);
+ List<ICaseResult> cases = module.getResults();
+ assertEquals("Expected 1 test case", 1, cases.size());
+ ICaseResult case1 = cases.get(0);
+ List<ITestResult> testResults = case1.getResults();
+ assertEquals("Expected 2 tests", 2, testResults.size());
+
+ long startTime = mReporter.getResult().getStartTime();
+ String expectedRunHistory =
+ String.format(
+ "[{\"startTime\":1,\"endTime\":2},"
+ + "{\"startTime\":3,\"endTime\":4},{\"startTime\":%d,\"endTime\":%d}]",
+ startTime, startTime + 10);
+ assertEquals(expectedRunHistory, invocationResult.getInvocationInfo().get("run_history"));
+
+ // Test 1 details
+ ITestResult finalTestResult1 = case1.getResult(METHOD_1);
+ assertNotNull(String.format("Expected result for %s", TEST_1), finalTestResult1);
+ assertEquals(
+ String.format("Expected fail for %s", TEST_1),
+ TestStatus.FAIL,
+ finalTestResult1.getResultStatus());
+ assertEquals(finalTestResult1.getStackTrace(), STACK_TRACE);
+
+ // Test 2 details
+ ITestResult finalTestResult2 = case1.getResult(METHOD_2);
+ assertNotNull(String.format("Expected result for %s", TEST_2), finalTestResult2);
+ assertEquals(
+ String.format("Expected pass for %s", TEST_2),
+ TestStatus.PASS,
+ finalTestResult2.getResultStatus());
+ }
}
diff --git a/common/util/src/com/android/compatibility/common/util/InvocationResult.java b/common/util/src/com/android/compatibility/common/util/InvocationResult.java
index 45780a9..f4f294e 100644
--- a/common/util/src/com/android/compatibility/common/util/InvocationResult.java
+++ b/common/util/src/com/android/compatibility/common/util/InvocationResult.java
@@ -17,6 +17,7 @@
import java.io.File;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,6 +31,14 @@
*/
public class InvocationResult implements IInvocationResult {
+ /** Helper object for JSON conversion. */
+ public static final class RunHistory {
+ public long startTime;
+ public long endTime;
+ }
+
+ private Collection<RunHistory> mRunHistories = new ArrayList<>();
+
private long mTimestamp;
private Map<String, IModuleResult> mModuleResults = new LinkedHashMap<>();
private Map<String, String> mInvocationInfo = new HashMap<>();
@@ -40,6 +49,11 @@
private RetryChecksumStatus mRetryChecksumStatus = RetryChecksumStatus.NotRetry;
private File mRetryDirectory = null;
+ /** @return a collection of the run history of previous runs. */
+ public Collection<RunHistory> getRunHistories() {
+ return mRunHistories;
+ }
+
/**
* {@inheritDoc}
*/
diff --git a/common/util/src/com/android/compatibility/common/util/ResultHandler.java b/common/util/src/com/android/compatibility/common/util/ResultHandler.java
index 1be3624..53f73d3 100644
--- a/common/util/src/com/android/compatibility/common/util/ResultHandler.java
+++ b/common/util/src/com/android/compatibility/common/util/ResultHandler.java
@@ -37,6 +37,7 @@
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
@@ -104,6 +105,9 @@
private static final String RESULT_ATTR = "result";
private static final String RESULT_TAG = "Result";
private static final String RUNTIME_ATTR = "runtime";
+ private static final String RUN_HISTORY_ATTR = "run_history";
+ private static final String RUN_HISTORY_TAG = "RunHistory";
+ private static final String RUN_TAG = "Run";
private static final String SCREENSHOT_TAG = "Screenshot";
private static final String SKIPPED_ATTR = "skipped";
private static final String STACK_TAG = "StackTrace";
@@ -198,6 +202,10 @@
invocation.addInvocationInfo(BUILD_ID, parser.getAttributeValue(NS, BUILD_ID));
invocation.addInvocationInfo(BUILD_PRODUCT, parser.getAttributeValue(NS,
BUILD_PRODUCT));
+ String runHistoryValue = parser.getAttributeValue(NS, RUN_HISTORY_ATTR);
+ if (runHistoryValue != null) {
+ invocation.addInvocationInfo(RUN_HISTORY_ATTR, runHistoryValue);
+ }
// The build fingerprint needs to reflect the true fingerprint of the device under test,
// ignoring potential overrides made by test suites (namely STS) for APFE build
@@ -212,7 +220,19 @@
// --skip-device-info flag
parser.nextTag();
parser.require(XmlPullParser.END_TAG, NS, BUILD_TAG);
+
+ // Parse RunHistory tag.
parser.nextTag();
+ boolean hasRunHistoryTag = true;
+ try {
+ parser.require(parser.START_TAG, NS, RUN_HISTORY_TAG);
+ } catch (XmlPullParserException e) {
+ hasRunHistoryTag = false;
+ }
+ if (hasRunHistoryTag) {
+ parseRunHistory(parser);
+ }
+
parser.require(XmlPullParser.START_TAG, NS, SUMMARY_TAG);
parser.nextTag();
parser.require(XmlPullParser.END_TAG, NS, SUMMARY_TAG);
@@ -308,6 +328,18 @@
}
}
+ /** Parse and replay all run history information. */
+ private static void parseRunHistory(XmlPullParser parser)
+ throws IOException, XmlPullParserException {
+ while (parser.nextTag() == XmlPullParser.START_TAG) {
+ parser.require(XmlPullParser.START_TAG, NS, RUN_TAG);
+ parser.nextTag();
+ parser.require(XmlPullParser.END_TAG, NS, RUN_TAG);
+ }
+ parser.require(XmlPullParser.END_TAG, NS, RUN_HISTORY_TAG);
+ parser.nextTag();
+ }
+
/**
* @param result
* @param resultDir
@@ -392,6 +424,21 @@
}
serializer.endTag(NS, BUILD_TAG);
+ // Run history - this contains a list of start and end times of previous runs. More
+ // information may be added in the future.
+ Collection<InvocationResult.RunHistory> runHistories =
+ ((InvocationResult) result).getRunHistories();
+ if (!runHistories.isEmpty()) {
+ serializer.startTag(NS, RUN_HISTORY_TAG);
+ for (InvocationResult.RunHistory runHistory : runHistories) {
+ serializer.startTag(NS, RUN_TAG);
+ serializer.attribute(NS, START_TIME_ATTR, String.valueOf(runHistory.startTime));
+ serializer.attribute(NS, END_TIME_ATTR, String.valueOf(runHistory.endTime));
+ serializer.endTag(NS, RUN_TAG);
+ }
+ serializer.endTag(NS, RUN_HISTORY_TAG);
+ }
+
// Summary
serializer.startTag(NS, SUMMARY_TAG);
serializer.attribute(NS, PASS_ATTR, Integer.toString(passed));
diff --git a/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java b/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
index 50a00f0..11ca722 100644
--- a/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
+++ b/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
@@ -18,18 +18,29 @@
import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.AbiUtils;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
import junit.framework.TestCase;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.StringReader;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+
/**
* Unit tests for {@link ResultHandler}
*/
@@ -59,10 +70,14 @@
private static final String BUILD_FINGERPRINT_UNALTERED = "build_fingerprint_unaltered";
private static final String BUILD_ID = "build_id";
private static final String BUILD_PRODUCT = "build_product";
+ private static final String RUN_HISTORY = "run_history";
private static final String EXAMPLE_BUILD_ID = "XYZ";
private static final String EXAMPLE_BUILD_PRODUCT = "wolverine";
private static final String EXAMPLE_BUILD_FINGERPRINT = "example_build_fingerprint";
private static final String EXAMPLE_BUILD_FINGERPRINT_UNALTERED = "example_build_fingerprint_unaltered";
+ private static final String EXAMPLE_RUN_HISTORY =
+ "[{\"startTime\":10000000000000,\"endTime\":10000000000001},"
+ + "{\"startTime\":10000000000002,\"endTime\":10000000000003}]";
private static final String DEVICE_A = "device123";
private static final String DEVICE_B = "device456";
@@ -176,6 +191,18 @@
result.addInvocationInfo(BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT);
result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID);
result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT);
+ result.addInvocationInfo(RUN_HISTORY, EXAMPLE_RUN_HISTORY);
+ Collection<InvocationResult.RunHistory> runHistories =
+ ((InvocationResult) result).getRunHistories();
+ InvocationResult.RunHistory runHistory1 = new InvocationResult.RunHistory();
+ runHistory1.startTime = 10000000000000L;
+ runHistory1.endTime = 10000000000001L;
+ runHistories.add(runHistory1);
+ InvocationResult.RunHistory runHistory2 = new InvocationResult.RunHistory();
+ runHistory2.startTime = 10000000000002L;
+ runHistory2.endTime = 10000000000003L;
+ runHistories.add(runHistory2);
+
// Module A: test1 passes, test2 not executed
IModuleResult moduleA = result.getOrCreateModule(ID_A);
moduleA.setDone(false);
@@ -208,12 +235,31 @@
moduleBTest5.skipped();
// Serialize to file
- ResultHandler.writeResults(SUITE_NAME, SUITE_VERSION, SUITE_PLAN, SUITE_BUILD,
- result, resultDir, START_MS, END_MS, REFERENCE_URL, LOG_URL,
- COMMAND_LINE_ARGS);
+ File res =
+ ResultHandler.writeResults(
+ SUITE_NAME,
+ SUITE_VERSION,
+ SUITE_PLAN,
+ SUITE_BUILD,
+ result,
+ resultDir,
+ START_MS,
+ END_MS,
+ REFERENCE_URL,
+ LOG_URL,
+ COMMAND_LINE_ARGS);
+ String content = FileUtil.readStringFromFile(res);
+ assertXmlContainsAttribute(content, "Result/Build", "run_history", EXAMPLE_RUN_HISTORY);
+ assertXmlContainsNode(content, "Result/RunHistory");
+ assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000000");
+ assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000000001");
+ assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000002");
+ assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000000003");
// Parse the results and assert correctness
- checkResult(ResultHandler.getResultFromDir(resultDir), false);
+ result = ResultHandler.getResultFromDir(resultDir);
+ checkResult(result, false);
+ checkRunHistory(result);
}
public void testParsing() throws Exception {
@@ -350,6 +396,11 @@
checkResult(result, EXAMPLE_BUILD_FINGERPRINT, newTestFormat);
}
+ static void checkRunHistory(IInvocationResult result) {
+ Map<String, String> buildInfo = result.getInvocationInfo();
+ assertEquals("Incorrect run history", EXAMPLE_RUN_HISTORY, buildInfo.get(RUN_HISTORY));
+ }
+
static void checkResult(
IInvocationResult result, String expectedBuildFingerprint, boolean newTestFormat)
throws Exception {
@@ -450,4 +501,55 @@
assertNull("Unexpected stack trace", moduleBTest5.getStackTrace());
assertNull("Unexpected report", moduleBTest5.getReportLog());
}
+
+ /** Return all XML nodes that match the given xPathExpression. */
+ private NodeList getXmlNodes(String xml, String xPathExpression)
+ throws XPathExpressionException {
+
+ InputSource inputSource = new InputSource(new StringReader(xml));
+ XPath xpath = XPathFactory.newInstance().newXPath();
+ return (NodeList) xpath.evaluate(xPathExpression, inputSource, XPathConstants.NODESET);
+ }
+
+ /** Assert that the XML contains a node matching the given xPathExpression. */
+ private NodeList assertXmlContainsNode(String xml, String xPathExpression)
+ throws XPathExpressionException {
+ NodeList nodes = getXmlNodes(xml, xPathExpression);
+ assertNotNull(
+ String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression),
+ nodes);
+ assertTrue(
+ String.format(
+ "XML '%s' should have returned at least 1 node for xpath '%s', "
+ + "but returned %s nodes instead.",
+ xml, xPathExpression, nodes.getLength()),
+ nodes.getLength() >= 1);
+ return nodes;
+ }
+
+ /**
+ * Assert that the XML contains a node matching the given xPathExpression and that the node has
+ * a given value.
+ */
+ private void assertXmlContainsAttribute(
+ String xml, String xPathExpression, String attributeName, String attributeValue)
+ throws XPathExpressionException {
+ NodeList nodes = assertXmlContainsNode(xml, xPathExpression);
+ boolean found = false;
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+ String value = element.getAttribute(attributeName);
+ if (attributeValue.equals(value)) {
+ found = true;
+ break;
+ }
+ }
+
+ assertTrue(
+ String.format(
+ "xPath '%s' should contain attribute '%s' but does not. XML: '%s'",
+ xPathExpression, attributeName, xml),
+ found);
+ }
}
