Add summary xml file to CTS results output
Test: Built and executed full CTS suite.
Change-Id: I9cefa6a0927cccf8821fb1937186a2e586855d43
diff --git a/common/host-side/tradefed/res/report/compatibility_failures.xsl b/common/host-side/tradefed/res/report/compatibility_failures.xsl
new file mode 100644
index 0000000..be65b91
--- /dev/null
+++ b/common/host-side/tradefed/res/report/compatibility_failures.xsl
@@ -0,0 +1,270 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2016 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE xsl:stylesheet [ <!ENTITY nbsp " "> ]>
+<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+ <xsl:output method="html" version="1.0" encoding="UTF-8" indent="yes"/>
+
+ <xsl:template match="/">
+
+ <html>
+ <head>
+ <title>Test Report</title>
+ <style type="text/css">
+ @import "compatibility_result.css";
+ </style>
+ </head>
+ <body>
+ <div>
+ <table class="title">
+ <tr>
+ <td align="left"><img src="logo.png"/></td>
+ </tr>
+ </table>
+ </div>
+
+ <div>
+ <table class="summary">
+ <tr>
+ <th colspan="2">Summary</th>
+ </tr>
+ <tr>
+ <td class="rowtitle">Suite / Plan</td>
+ <td>
+ <xsl:value-of select="Result/@suite_name"/> / <xsl:value-of select="Result/@suite_plan"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Suite / Build</td>
+ <td>
+ <xsl:value-of select="Result/@suite_version"/> / <xsl:value-of select="Result/@suite_build_number"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Host Info</td>
+ <td>
+ Result/@start
+ <xsl:value-of select="Result/@host_name"/>
+ (<xsl:value-of select="Result/@os_name"/> - <xsl:value-of select="Result/@os_version"/>)
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Start time / End Time</td>
+ <td>
+ <xsl:value-of select="Result/@start_display"/> /
+ <xsl:value-of select="Result/@end_display"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Tests Passed</td>
+ <td>
+ <xsl:value-of select="Result/Summary/@pass"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Tests Failed</td>
+ <td>
+ <xsl:value-of select="Result/Summary/@failed"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Tests Not Executed</td>
+ <td>
+ <xsl:value-of select="Result/Summary/@not_executed"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Modules Done</td>
+ <td>
+ <xsl:value-of select="Result/Summary/@modules_done"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Modules Total</td>
+ <td>
+ <xsl:value-of select="Result/Summary/@modules_total"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Fingerprint</td>
+ <td>
+ <xsl:value-of select="Result/Build/@build_fingerprint"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Security Patch</td>
+ <td>
+ <xsl:value-of select="Result/Build/@build_version_security_patch"/>
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">Release (SDK)</td>
+ <td>
+ <xsl:value-of select="Result/Build/@build_version_release"/> (<xsl:value-of select="Result/Build/@build_version_sdk"/>)
+ </td>
+ </tr>
+ <tr>
+ <td class="rowtitle">ABIs</td>
+ <td>
+ <xsl:value-of select="Result/Build/@build_abis"/>
+ </td>
+ </tr>
+ </table>
+ </div>
+
+ <!-- High level summary of test execution -->
+ <br/>
+ <div>
+ <table class="testsummary">
+ <tr>
+ <th>Module</th>
+ <th>Passed</th>
+ <th>Failed</th>
+ <th>Not Executed</th>
+ <th>Total Tests</th>
+ </tr>
+ <xsl:for-each select="Result/Module">
+ <tr>
+ <td>
+ <xsl:if test="count(TestCase/Test[@result = 'fail']) > 0">
+ <xsl:variable name="href"><xsl:value-of select="@name"/> - <xsl:value-of select="@abi"/></xsl:variable>
+ <a href="#{$href}"><xsl:value-of select="@name"/> - <xsl:value-of select="@abi"/></a>
+ </xsl:if>
+ <xsl:if test="count(TestCase/Test[@result = 'fail']) < 1">
+ <xsl:value-of select="@name"/> - <xsl:value-of select="@abi"/>
+ </xsl:if>
+ </td>
+ <td>
+ <xsl:value-of select="@pass"/>
+ </td>
+ <td>
+ <xsl:value-of select="count(TestCase/Test[@result = 'fail'])"/>
+ </td>
+ <td>
+ <xsl:value-of select="count(TestCase/Test[@result = 'not_executed'])"/>
+ </td>
+ <td>
+ <xsl:value-of select="count(TestCase/Test[@result = 'fail']) + @pass + count(TestCase/Test[@result = 'not_executed']) "/>
+ </td>
+ </tr>
+ </xsl:for-each> <!-- end Module -->
+ </table>
+ </div>
+
+ <br/>
+ <xsl:call-template name="detailedTestReport">
+ <xsl:with-param name="resultFilter" select="'fail'" />
+ </xsl:call-template>
+
+ </body>
+ </html>
+ </xsl:template>
+
+ <xsl:template name="detailedTestReport">
+ <xsl:param name="resultFilter" />
+ <div>
+ <xsl:for-each select="Result/Module">
+ <xsl:if test="$resultFilter=''
+ or count(TestCase/Test[@result=$resultFilter]) > 0">
+
+ <table class="testdetails">
+ <tr>
+ <td class="module" colspan="3">
+ <xsl:variable name="href"><xsl:value-of select="@name"/> - <xsl:value-of select="@abi"/></xsl:variable>
+ <a name="{$href}"><xsl:value-of select="@name"/> - <xsl:value-of select="@abi"/></a>
+ </td>
+ </tr>
+
+ <tr>
+ <th width="30%">Test</th>
+ <th width="5%">Result</th>
+ <th>Details</th>
+ </tr>
+
+ <xsl:for-each select="TestCase">
+ <xsl:variable name="TestCase" select="."/>
+ <!-- test -->
+ <xsl:for-each select="Test">
+ <xsl:if test="$resultFilter='' or @result=$resultFilter">
+ <tr>
+ <td class="testname"> <xsl:value-of select="$TestCase/@name"/>#<xsl:value-of select="@name"/></td>
+
+ <!-- test results -->
+ <xsl:if test="@result='pass'">
+ <td class="pass">
+ <div style="text-align: center; margin-left:auto; margin-right:auto;">
+ <xsl:value-of select="@result"/>
+ </div>
+ </td>
+ <td class="failuredetails"/>
+ </xsl:if>
+
+ <xsl:if test="@result='fail'">
+ <td class="failed">
+ <div style="text-align: center; margin-left:auto; margin-right:auto;">
+ <xsl:value-of select="@result"/>
+ </div>
+ </td>
+ <td class="failuredetails">
+ <div class="details">
+ <xsl:value-of select="Failure/@message"/>
+ </div>
+ </td>
+ </xsl:if>
+
+ <xsl:if test="@result='not_executed'">
+ <td class="not_executed">
+ <div style="text-align: center; margin-left:auto; margin-right:auto;">
+ <xsl:value-of select="@result"/>
+ </div>
+ </td>
+ <td class="failuredetails"></td>
+ </xsl:if>
+ </tr> <!-- finished with a row -->
+ </xsl:if>
+ </xsl:for-each> <!-- end test -->
+ </xsl:for-each>
+ </table>
+ </xsl:if>
+ </xsl:for-each> <!-- end test Module -->
+ </div>
+ </xsl:template>
+
+ <!-- Take a delimited string and insert line breaks after a some number of elements. -->
+ <xsl:template name="formatDelimitedString">
+ <xsl:param name="string" />
+ <xsl:param name="numTokensPerRow" select="10" />
+ <xsl:param name="tokenIndex" select="1" />
+ <xsl:if test="$string">
+ <!-- Requires the last element to also have a delimiter after it. -->
+ <xsl:variable name="token" select="substring-before($string, ';')" />
+ <xsl:value-of select="$token" />
+ <xsl:text> </xsl:text>
+
+ <xsl:if test="$tokenIndex mod $numTokensPerRow = 0">
+ <br />
+ </xsl:if>
+
+ <xsl:call-template name="formatDelimitedString">
+ <xsl:with-param name="string" select="substring-after($string, ';')" />
+ <xsl:with-param name="numTokensPerRow" select="$numTokensPerRow" />
+ <xsl:with-param name="tokenIndex" select="$tokenIndex + 1" />
+ </xsl:call-template>
+ </xsl:if>
+ </xsl:template>
+
+</xsl:stylesheet>
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
index fa8d8ad..4c21f08 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
@@ -16,7 +16,6 @@
package com.android.compatibility.common.tradefed.result;
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
-import com.android.compatibility.common.tradefed.result.InvocationFailureHandler;
import com.android.compatibility.common.tradefed.testtype.CompatibilityTest;
import com.android.compatibility.common.util.ICaseResult;
import com.android.compatibility.common.util.IInvocationResult;
@@ -28,7 +27,6 @@
import com.android.compatibility.common.util.ResultHandler;
import com.android.compatibility.common.util.ResultUploader;
import com.android.compatibility.common.util.TestStatus;
-import com.android.ddmlib.Log;
import com.android.ddmlib.Log.LogLevel;
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.tradefed.build.IBuildInfo;
@@ -59,13 +57,10 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.text.SimpleDateFormat;
import java.util.Collections;
-import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
/**
@@ -468,8 +463,15 @@
mBuildHelper.getSuiteBuild(), mResult, mResultDir, startTime,
elapsedTime + startTime, mReferenceUrl, getLogUrl(),
mBuildHelper.getCommandLineArgs());
- info("Test Result: %s", resultFile.getCanonicalPath());
File zippedResults = zipResults(mResultDir);
+
+ // Create failure report after zip file so extra data is not uploaded
+ File failureReport = ResultHandler.createFailureReport(resultFile);
+ if (failureReport.exists()) {
+ info("Test Result: %s", failureReport.getCanonicalPath());
+ } else {
+ info("Test Result: %s", resultFile.getCanonicalPath());
+ }
info("Full Result: %s", zippedResults.getCanonicalPath());
saveLog(resultFile, zippedResults);
diff --git a/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java b/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
index d04ed8e..b3ca245 100644
--- a/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
+++ b/common/host-side/util/src/com/android/compatibility/common/util/ResultHandler.java
@@ -29,6 +29,7 @@
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.UnknownHostException;
@@ -44,6 +45,12 @@
import java.util.Map.Entry;
import java.util.Set;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
/**
* Handles conversion of results to/from files.
*/
@@ -54,6 +61,8 @@
private static final String NS = null;
private static final String RESULT_FILE_VERSION = "5.0";
/* package */ static final String TEST_RESULT_FILE_NAME = "test_result.xml";
+ private static final String FAILURE_REPORT_NAME = "test_result_failures.html";
+ private static final String FAILURE_XSL_FILE_NAME = "compatibility_failures.xsl";
// XML constants
private static final String ABI_ATTR = "abi";
@@ -115,7 +124,7 @@
public static List<IInvocationResult> getResults(
File resultsDir, Boolean useChecksum) {
List<IInvocationResult> results = new ArrayList<>();
- List<File> files = getResultDirectories(resultsDir);;
+ List<File> files = getResultDirectories(resultsDir);
for (File resultDir : files) {
if (!resultDir.isDirectory()) {
continue;
@@ -270,7 +279,7 @@
String suiteBuild, IInvocationResult result, File resultDir,
long startTime, long endTime, String referenceUrl, String logUrl,
String commandLineArgs)
- throws IOException, XmlPullParserException {
+ throws IOException, XmlPullParserException {
int passed = result.countResults(TestStatus.PASS);
int failed = result.countResults(TestStatus.FAIL);
int notExecuted = result.getNotExecuted();
@@ -321,7 +330,8 @@
String hostName = "";
try {
hostName = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException ignored) {}
+ } catch (UnknownHostException ignored) {
+ }
serializer.attribute(NS, HOST_NAME_ATTR, hostName);
serializer.attribute(NS, OS_NAME_ATTR, System.getProperty("os.name"));
serializer.attribute(NS, OS_VERSION_ATTR, System.getProperty("os.version"));
@@ -359,6 +369,8 @@
serializer.attribute(NS, RUNTIME_ATTR, String.valueOf(module.getRuntime()));
serializer.attribute(NS, DONE_ATTR, Boolean.toString(module.isDone()));
serializer.attribute(NS, NOT_EXECUTED_ATTR, Integer.toString(module.getNotExecuted()));
+ serializer.attribute(NS, PASS_ATTR,
+ Integer.toString(module.countResults(TestStatus.PASS)));
for (ICaseResult cr : module.getResults()) {
serializer.startTag(NS, CASE_TAG);
serializer.attribute(NS, NAME_ATTR, cr.getName());
@@ -415,6 +427,19 @@
return resultFile;
}
+ public static File createFailureReport(File inputXml) {
+ File failureReport = new File(inputXml.getParentFile(), FAILURE_REPORT_NAME);
+ try (InputStream xslStream = ResultHandler.class.getResourceAsStream(
+ String.format("/report/%s", FAILURE_XSL_FILE_NAME));
+ OutputStream outputStream = new FileOutputStream(failureReport)) {
+
+ Transformer transformer = TransformerFactory.newInstance().newTransformer(
+ new StreamSource(xslStream));
+ transformer.transform(new StreamSource(inputXml), new StreamResult(outputStream));
+ } catch (IOException | TransformerException ignored) { }
+ return failureReport;
+ }
+
private static void createChecksum(File resultDir, IInvocationResult invocationResult) {
RetryChecksumStatus retryStatus = invocationResult.getRetryChecksumStatus();
switch (retryStatus) {
