Add "Omitted" State for CTS Tests
Add a new "omitted" state for tests. This is different from "not
executed" to fix a problem where CTS would keep retrying to run
the not executed tests over and over again.
In a prior change, I switched the default test result from pass to
not executed in the test result parser, since I didn't want skipped
tests to be reported as passed (or tests that failed horribly to
report pass). This worked fine when running a specific package "-p"
but started to fail when running the entire plan "--plan CTS." This
is because there are two code paths where one infinitely retries
if there are any tests not marked as "not executed" but the other
just retries once. Tests with the new annotations thus caused an
infinite loop of retrying... By setting tests to "omitted" by
default, the tests won't be retried over and over.
Change-Id: I3b4995089605860b1ed5e6b752dde811132f16f8
diff --git a/tools/host/src/com/android/cts/ConsoleUi.java b/tools/host/src/com/android/cts/ConsoleUi.java
index 702631d..7ebf01a 100644
--- a/tools/host/src/com/android/cts/ConsoleUi.java
+++ b/tools/host/src/com/android/cts/ConsoleUi.java
@@ -82,6 +82,7 @@
mResultCodeMap.put(CtsTestResult.STR_ERROR, CtsTestResult.CODE_ERROR);
mResultCodeMap.put(CtsTestResult.STR_NOT_EXECUTED, CtsTestResult.CODE_NOT_EXECUTED);
mResultCodeMap.put(CtsTestResult.STR_TIMEOUT, CtsTestResult.CODE_TIMEOUT);
+ mResultCodeMap.put(CtsTestResult.STR_OMITTED, CtsTestResult.CODE_OMITTED);
}
public ConsoleUi(TestHost host) {
@@ -1202,7 +1203,7 @@
CUIOutputStream.println("List of all results: ");
CUIOutputStream.println("Session\t\tTest result\t\t\t\tStart time\t\tEnd time\t"
+ "\tProfile\tTest plan name\t");
- CUIOutputStream.println("\t\tPass\tFail\tTimeout\tNotExecuted");
+ CUIOutputStream.println("\t\tPass\tFail\tTimeout\tOmitted\tNotExecuted");
for (TestSession session : sessions) {
TestSessionLog log = session.getSessionLog();
@@ -1210,6 +1211,8 @@
CtsTestResult.CODE_PASS).size();
int failNum = log.getTestList(
CtsTestResult.CODE_FAIL).size();
+ int omittedNum = log.getTestList(
+ CtsTestResult.CODE_OMITTED).size();
int notExecutedNum = log.getTestList(
CtsTestResult.CODE_NOT_EXECUTED).size();
int timeOutNum = log.getTestList(
@@ -1217,6 +1220,7 @@
String resStr = Long.toString(passNum) + "\t" + failNum;
resStr += "\t" + timeOutNum;
+ resStr += "\t" + omittedNum;
resStr += "\t" + notExecutedNum;
String startTimeStr =
diff --git a/tools/host/src/com/android/cts/CtsTestResult.java b/tools/host/src/com/android/cts/CtsTestResult.java
index a02c445..0aea74b3 100644
--- a/tools/host/src/com/android/cts/CtsTestResult.java
+++ b/tools/host/src/com/android/cts/CtsTestResult.java
@@ -37,12 +37,14 @@
public static final int CODE_FAIL = 2;
public static final int CODE_ERROR = 3;
public static final int CODE_TIMEOUT = 4;
+ public static final int CODE_OMITTED = 5;
public static final int CODE_FIRST = CODE_INIT;
- public static final int CODE_LAST = CODE_TIMEOUT;
+ public static final int CODE_LAST = CODE_OMITTED;
public static final String STR_ERROR = "error";
public static final String STR_TIMEOUT = "timeout";
public static final String STR_NOT_EXECUTED = "notExecuted";
+ public static final String STR_OMITTED = "omitted";
public static final String STR_FAIL = "fail";
public static final String STR_PASS = "pass";
@@ -55,6 +57,7 @@
sCodeToResultMap.put(CODE_FAIL, STR_FAIL);
sCodeToResultMap.put(CODE_ERROR, STR_ERROR);
sCodeToResultMap.put(CODE_TIMEOUT, STR_TIMEOUT);
+ sCodeToResultMap.put(CODE_OMITTED, STR_OMITTED);
sResultToCodeMap = new HashMap<String, Integer>();
for (int code : sCodeToResultMap.keySet()) {
sResultToCodeMap.put(sCodeToResultMap.get(code), code);
diff --git a/tools/host/src/com/android/cts/TestDevice.java b/tools/host/src/com/android/cts/TestDevice.java
index 661dcdb..9c82af1 100644
--- a/tools/host/src/com/android/cts/TestDevice.java
+++ b/tools/host/src/com/android/cts/TestDevice.java
@@ -1230,7 +1230,7 @@
mResultLines = new ArrayList<String>();
mStackTrace = null;
mFailedMsg = null;
- mResultCode = CtsTestResult.CODE_NOT_EXECUTED;
+ mResultCode = CtsTestResult.CODE_OMITTED;
}
/** {@inheritDoc} */
diff --git a/tools/host/src/com/android/cts/TestSession.java b/tools/host/src/com/android/cts/TestSession.java
index cbd2aba..18ace44 100644
--- a/tools/host/src/com/android/cts/TestSession.java
+++ b/tools/host/src/com/android/cts/TestSession.java
@@ -487,14 +487,15 @@
private void displayTestResultSummary() {
int passNum = mSessionLog.getTestList(CtsTestResult.CODE_PASS).size();
int failNum = mSessionLog.getTestList(CtsTestResult.CODE_FAIL).size();
- int notExecutedNum =
- mSessionLog.getTestList(CtsTestResult.CODE_NOT_EXECUTED).size();
+ int omittedNum = mSessionLog.getTestList(CtsTestResult.CODE_OMITTED).size();
+ int notExecutedNum = mSessionLog.getTestList(CtsTestResult.CODE_NOT_EXECUTED).size();
int timeOutNum = mSessionLog.getTestList(CtsTestResult.CODE_TIMEOUT).size();
int total = passNum + failNum + notExecutedNum + timeOutNum;
println("Test summary: pass=" + passNum
+ " fail=" + failNum
+ " timeOut=" + timeOutNum
+ + " omitted=" + omittedNum
+ " notExecuted=" + notExecutedNum
+ " Total=" + total);
}
diff --git a/tools/host/src/com/android/cts/TestSessionLog.java b/tools/host/src/com/android/cts/TestSessionLog.java
index d3f836a..99a15bf 100644
--- a/tools/host/src/com/android/cts/TestSessionLog.java
+++ b/tools/host/src/com/android/cts/TestSessionLog.java
@@ -48,7 +48,7 @@
private static final String ATTRIBUTE_KNOWN_FAILURE = "KnownFailure";
public static final String CTS_RESULT_FILE_NAME = "testResult.xml";
- private static final String CTS_RESULT_FILE_VERSION = "1.6";
+ private static final String CTS_RESULT_FILE_VERSION = "1.7";
static final String ATTRIBUTE_STARTTIME = "starttime";
static final String ATTRIBUTE_ENDTIME = "endtime";
@@ -80,6 +80,7 @@
static final String ATTRIBUTE_PASS = "pass";
static final String ATTRIBUTE_FAILED = "failed";
static final String ATTRIBUTE_TIMEOUT = "timeout";
+ static final String ATTRIBUTE_OMITTED = "omitted";
static final String ATTRIBUTE_NOT_EXECUTED = "notExecuted";
static final String TAG_DEVICEINFO = "DeviceInfo";
@@ -386,12 +387,14 @@
int passNum = getTestList(CtsTestResult.CODE_PASS).size();
int failNum = getTestList(CtsTestResult.CODE_FAIL).size();
+ int omittedNum = getTestList(CtsTestResult.CODE_OMITTED).size();
int notExecutedNum = getTestList(CtsTestResult.CODE_NOT_EXECUTED).size();
int timeOutNum = getTestList(CtsTestResult.CODE_TIMEOUT).size();
Node summaryNode = doc.createElement(TAG_SUMMARY);
root.appendChild(summaryNode);
setAttribute(doc, summaryNode, ATTRIBUTE_PASS, passNum);
setAttribute(doc, summaryNode, ATTRIBUTE_FAILED, failNum);
+ setAttribute(doc, summaryNode, ATTRIBUTE_OMITTED, omittedNum);
setAttribute(doc, summaryNode, ATTRIBUTE_NOT_EXECUTED, notExecutedNum);
setAttribute(doc, summaryNode, ATTRIBUTE_TIMEOUT, timeOutNum);
diff --git a/tools/host/src/res/cts_result.css b/tools/host/src/res/cts_result.css
index b5b4009..d7ce510 100644
--- a/tools/host/src/res/cts_result.css
+++ b/tools/host/src/res/cts_result.css
@@ -175,13 +175,7 @@
margin-right:auto;
}
-td.timeout {
- background-color: #A5C639;
- vertical-align: top;
- text-align: center;
-}
-
-td.notExecuted {
+td.timeout, td.omitted, td.notExecuted {
background-color: #A5C639;
vertical-align: top;
text-align: center;
diff --git a/tools/host/src/res/cts_result.xsl b/tools/host/src/res/cts_result.xsl
index bd99c8d..758f7ea 100644
--- a/tools/host/src/res/cts_result.xsl
+++ b/tools/host/src/res/cts_result.xsl
@@ -302,6 +302,12 @@
</TD>
</TR>
<TR>
+ <TD class="rowtitle">Tests Omitted</TD>
+ <TD>
+ <xsl:value-of select="TestResult/Summary/@omitted"/>
+ </TD>
+ </TR>
+ <TR>
<TD class="rowtitle">Tests Not Executed</TD>
<TD>
<xsl:value-of select="TestResult/Summary/@notExecuted"/>
@@ -455,6 +461,15 @@
</TD>
</xsl:if>
+ <xsl:if test="@result='omitted'">
+ <TD class="omitted">
+ <div style="text-align: center; margin-left:auto; margin-right:auto;">
+ <xsl:value-of select="@result"/>
+ </div>
+ </TD>
+ <TD class="failuredetails"></TD>
+ </xsl:if>
+
<xsl:if test="@result='notExecuted'">
<TD class="notExecuted">
<div style="text-align: center; margin-left:auto; margin-right:auto;">
