cts: add TestLog tags to XML report
b/16959454
- When configured properly, TF can store logs and report the URLs where
they are stored back to the test runner. So capture those URLs and
write them to the XML report for future processing.
- CtsXmlResultReporter implements ILogSaver, which has a testLogSaved callback
that gives the URL of the test log file. The URL to the test log is included
in the test result XML when the test fails and when the feature is enabled.
- Add a new tag <TestLog> to represent selected logs that testLogSaved
tells us about. Put all logic regarding this tag into the TestLog
class. TestLogType is an enum that defines what specific logs we are
interested in and how to identify them by examining the data name
passed by the testLogSaved callback.
- Finally, add a "include-test-log-tags" flag to control whether these
tags are enabled in the XML. This has the possibility of doubling the
size of reports, so we will only enable this feature in the lab
setting.
Change-Id: Iecfa26abeee3a4d4d36dc95e5d0bfb5cac411236
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
index 08e9a0b..8cb4072 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
@@ -27,10 +27,13 @@
import com.android.tradefed.config.Option;
import com.android.tradefed.config.Option.Importance;
import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ILogSaver;
+import com.android.tradefed.result.ILogSaverListener;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.ITestSummaryListener;
import com.android.tradefed.result.InputStreamSource;
import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.LogFile;
import com.android.tradefed.result.LogFileSaver;
import com.android.tradefed.result.TestSummary;
import com.android.tradefed.util.FileUtil;
@@ -54,7 +57,9 @@
* <p/>
* Outputs xml in format governed by the cts_result.xsd
*/
-public class CtsXmlResultReporter implements ITestInvocationListener, ITestSummaryListener {
+public class CtsXmlResultReporter
+ implements ITestInvocationListener, ITestSummaryListener, ILogSaverListener {
+
private static final String LOG_TAG = "CtsXmlResultReporter";
static final String TEST_RESULT_FILE_NAME = "testResult.xml";
@@ -89,11 +94,15 @@
@Option(name = "result-server", description = "Server to publish test results.")
private String mResultServer;
+ @Option(name = "include-test-log-tags", description = "Include test log tags in XML report.")
+ private boolean mIncludeTestLogTags = false;
+
protected IBuildInfo mBuildInfo;
private String mStartTime;
private String mDeviceSerial;
private TestResults mResults = new TestResults();
private TestPackageResult mCurrentPkgResult = null;
+ private Test mCurrentTest = null;
private boolean mIsDeviceInfoRun = false;
private ResultReporter mReporter;
private File mLogDir;
@@ -104,6 +113,11 @@
mReportDir = reportDir;
}
+ /** Set whether to include TestLog tags in the XML reports. */
+ public void setIncludeTestLogTags(boolean include) {
+ mIncludeTestLogTags = include;
+ }
+
/**
* {@inheritDoc}
*/
@@ -211,6 +225,20 @@
}
/**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testLogSaved(String dataName, LogDataType dataType, InputStreamSource dataStream,
+ LogFile logFile) {
+ if (mIncludeTestLogTags && mCurrentTest != null) {
+ TestLog log = TestLog.fromDataName(dataName, logFile.getUrl());
+ if (log != null) {
+ mCurrentTest.addTestLog(log);
+ }
+ }
+ }
+
+ /**
* Return the {@link LogFileSaver} to use.
* <p/>
* Exposed for unit testing.
@@ -219,6 +247,10 @@
return new LogFileSaver(mLogDir);
}
+ @Override
+ public void setLogSaver(ILogSaver logSaver) {
+ // Don't need to keep a reference to logSaver, because we don't save extra logs in this class.
+ }
@Override
public void testRunStarted(String id, int numTests) {
@@ -235,7 +267,7 @@
@Override
public void testStarted(TestIdentifier test) {
if (!mIsDeviceInfoRun) {
- mCurrentPkgResult.insertTest(test);
+ mCurrentTest = mCurrentPkgResult.insertTest(test);
}
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java
index 023cfcb..12a2b29 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java
@@ -16,12 +16,15 @@
package com.android.cts.tradefed.result;
import com.android.ddmlib.Log;
+import com.android.cts.tradefed.result.TestLog.TestLogType;
import org.kxml2.io.KXmlSerializer;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
/**
* Data structure that represents a "Test" result XML element.
@@ -58,6 +61,12 @@
private String mDetails;
/**
+ * Log info for this test like a logcat dump or bugreport.
+ * Use *Locked methods instead of mutating this directly.
+ */
+ private List<TestLog> mTestLogs;
+
+ /**
* Create an empty {@link Test}
*/
public Test() {
@@ -76,6 +85,13 @@
}
/**
+ * Add a test log to this Test.
+ */
+ public void addTestLog(TestLog testLog) {
+ addTestLogLocked(testLog);
+ }
+
+ /**
* Set the name of this {@link Test}
*/
public void setName(String name) {
@@ -157,6 +173,8 @@
serializer.attribute(CtsXmlResultReporter.ns, STARTTIME_ATTR, mStartTime);
serializer.attribute(CtsXmlResultReporter.ns, ENDTIME_ATTR, mEndTime);
+ serializeTestLogsLocked(serializer);
+
if (mMessage != null) {
serializer.startTag(CtsXmlResultReporter.ns, SCENE_TAG);
serializer.attribute(CtsXmlResultReporter.ns, MESSAGE_ATTR, mMessage);
@@ -328,10 +346,38 @@
mMessage = getAttribute(parser, MESSAGE_ATTR);
} else if (eventType == XmlPullParser.START_TAG && parser.getName().equals(STACK_TAG)) {
mStackTrace = parser.nextText();
+ } else if (eventType == XmlPullParser.START_TAG && TestLog.isTag(parser.getName())) {
+ parseTestLog(parser);
} else if (eventType == XmlPullParser.END_TAG && parser.getName().equals(TAG)) {
return;
}
eventType = parser.next();
}
}
+
+ /** Parse a TestLog entry from the parser positioned at a TestLog tag. */
+ private void parseTestLog(XmlPullParser parser) throws XmlPullParserException{
+ TestLog log = TestLog.fromXml(parser);
+ if (log == null) {
+ throw new XmlPullParserException("invalid XML: bad test log tag");
+ }
+ addTestLog(log);
+ }
+
+ /** Add a TestLog to the test in a thread safe manner. */
+ private synchronized void addTestLogLocked(TestLog testLog) {
+ if (mTestLogs == null) {
+ mTestLogs = new ArrayList<>(TestLogType.values().length);
+ }
+ mTestLogs.add(testLog);
+ }
+
+ /** Serialize the TestLogs of this test in a thread safe manner. */
+ private synchronized void serializeTestLogsLocked(KXmlSerializer serializer) throws IOException {
+ if (mTestLogs != null) {
+ for (TestLog log : mTestLogs) {
+ log.serialize(serializer);
+ }
+ }
+ }
}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/TestLog.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestLog.java
new file mode 100644
index 0000000..1210432
--- /dev/null
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/TestLog.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.result;
+
+import org.kxml2.io.KXmlSerializer;
+import org.xmlpull.v1.XmlPullParser;
+
+import java.io.IOException;
+
+import javax.annotation.Nullable;
+
+/**
+ * TestLog describes a log for a test. It corresponds to the "TestLog" XML element.
+ */
+class TestLog {
+
+ private static final String TAG = "TestLog";
+ private static final String TYPE_ATTR = "type";
+ private static final String URL_ATTR = "url";
+
+ /** Type of log. */
+ public enum TestLogType {
+ LOGCAT("logcat-"),
+ BUGREPORT("bug-"),
+
+ ;
+
+ // This enum restricts the type of logs reported back to the server,
+ // because we do not intend to support them all. Using an enum somewhat
+ // assures that we will only see these expected types on the server side.
+
+ /**
+ * Returns the TestLogType from an ILogSaver data name or null
+ * if the data name is not supported.
+ */
+ @Nullable
+ static TestLogType fromDataName(String dataName) {
+ if (dataName == null) {
+ return null;
+ }
+
+ for (TestLogType type : values()) {
+ if (dataName.startsWith(type.dataNamePrefix)) {
+ return type;
+ }
+ }
+ return null;
+ }
+
+ private final String dataNamePrefix;
+
+ private TestLogType(String dataNamePrefix) {
+ this.dataNamePrefix = dataNamePrefix;
+ }
+
+ String getAttrValue() {
+ return name().toLowerCase();
+ }
+ }
+
+ /** Type of the log like LOGCAT or BUGREPORT. */
+ private final TestLogType mLogType;
+
+ /** Url pointing to the log file. */
+ private final String mUrl;
+
+ /** Create a TestLog from an ILogSaver callback. */
+ @Nullable
+ static TestLog fromDataName(String dataName, String url) {
+ TestLogType logType = TestLogType.fromDataName(dataName);
+ if (logType == null) {
+ return null;
+ }
+
+ if (url == null) {
+ return null;
+ }
+
+ return new TestLog(logType, url);
+ }
+
+ /** Create a TestLog from XML given a XmlPullParser positioned at the TestLog tag. */
+ @Nullable
+ static TestLog fromXml(XmlPullParser parser) {
+ String type = parser.getAttributeValue(null, TYPE_ATTR);
+ if (type == null) {
+ return null;
+ }
+
+ String url = parser.getAttributeValue(null, URL_ATTR);
+ if (url == null) {
+ return null;
+ }
+
+ try {
+ TestLogType logType = TestLogType.valueOf(type.toUpperCase());
+ return new TestLog(logType, url);
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ }
+
+ /** Create a TestLog directly given a log type and url. */
+ public static TestLog of(TestLogType logType, String url) {
+ return new TestLog(logType, url);
+ }
+
+ private TestLog(TestLogType logType, String url) {
+ this.mLogType = logType;
+ this.mUrl = url;
+ }
+
+ /** Returns this TestLog's log type. */
+ TestLogType getLogType() {
+ return mLogType;
+ }
+
+ /** Returns this TestLog's URL. */
+ String getUrl() {
+ return mUrl;
+ }
+
+ /** Serialize the TestLog to XML. */
+ public void serialize(KXmlSerializer serializer) throws IOException {
+ serializer.startTag(CtsXmlResultReporter.ns, TAG);
+ serializer.attribute(CtsXmlResultReporter.ns, TYPE_ATTR, mLogType.getAttrValue());
+ serializer.attribute(CtsXmlResultReporter.ns, URL_ATTR, mUrl);
+ serializer.endTag(CtsXmlResultReporter.ns, TAG);
+ }
+
+ /** Returns true if the tag is a TestLog tag. */
+ public static boolean isTag(String tagName) {
+ return TAG.equals(tagName);
+ }
+}
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
index 22dd6d9..ad1430e 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java
@@ -21,6 +21,7 @@
import com.android.cts.tradefed.result.TestResultsTest;
import com.android.cts.tradefed.result.TestSummaryXmlTest;
import com.android.cts.tradefed.result.TestTest;
+import com.android.cts.tradefed.result.TestLogTest;
import com.android.cts.tradefed.testtype.Abi;
import com.android.cts.tradefed.testtype.CtsTestTest;
import com.android.cts.tradefed.testtype.DeqpTestRunnerTest;
@@ -55,6 +56,7 @@
addTestSuite(TestResultsTest.class);
addTestSuite(TestSummaryXmlTest.class);
addTestSuite(TestTest.class);
+ addTestSuite(TestLogTest.class);
// testtype package
addTestSuite(CtsTestTest.class);
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java
index b74e26c..ae4a41e 100644
--- a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java
@@ -22,6 +22,8 @@
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.tradefed.build.IFolderBuildInfo;
import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.LogFile;
import com.android.tradefed.result.TestSummary;
import com.android.tradefed.result.XmlResultReporter;
import com.android.tradefed.util.FileUtil;
@@ -162,6 +164,8 @@
mResultReporter.testFailed(testId, trace);
mResultReporter.testEnded(testId, emptyMap);
mResultReporter.testRunEnded(3, emptyMap);
+ mResultReporter.testLogSaved("logcat-foo-bar", LogDataType.TEXT, null,
+ new LogFile("path", "url"));
mResultReporter.invocationEnded(1);
String output = getOutput();
// TODO: consider doing xml based compare
@@ -171,6 +175,37 @@
"<FailedScene message=\"this is a trace more trace\"> " +
"<StackTrace>this is a tracemore traceyet more trace</StackTrace>";
assertTrue(output.contains(failureTag));
+
+ // Check that no TestLog tags were added, because the flag wasn't enabled.
+ final String testLogTag = String.format("<TestLog type=\"logcat\" url=\"url\" />");
+ assertFalse(output, output.contains(testLogTag));
+ }
+
+ /**
+ * Test that flips the include-test-log-tags flag and checks that logs are written to the XML.
+ */
+ public void testIncludeTestLogTags() {
+ Map<String, String> emptyMap = Collections.emptyMap();
+ final TestIdentifier testId = new TestIdentifier("FooTest", "testFoo");
+ final String trace = "this is a trace\nmore trace\nyet more trace";
+
+ // Include TestLogTags in the XML.
+ mResultReporter.setIncludeTestLogTags(true);
+
+ mResultReporter.invocationStarted(mMockBuild);
+ mResultReporter.testRunStarted(AbiUtils.createId(UnitTests.ABI.getName(), "run"), 1);
+ mResultReporter.testStarted(testId);
+ mResultReporter.testFailed(testId, trace);
+ mResultReporter.testEnded(testId, emptyMap);
+ mResultReporter.testRunEnded(3, emptyMap);
+ mResultReporter.testLogSaved("logcat-foo-bar", LogDataType.TEXT, null,
+ new LogFile("path", "url"));
+ mResultReporter.invocationEnded(1);
+
+ // Check for TestLog tags because the flag was enabled via setIncludeTestLogTags.
+ final String output = getOutput();
+ final String testLogTag = String.format("<TestLog type=\"logcat\" url=\"url\" />");
+ assertTrue(output, output.contains(testLogTag));
}
public void testDeviceSetup() {
diff --git a/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestLogTest.java b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestLogTest.java
new file mode 100644
index 0000000..55c3071
--- /dev/null
+++ b/tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestLogTest.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.result;
+
+import com.android.cts.tradefed.result.TestLog.TestLogType;
+
+import org.kxml2.io.KXmlSerializer;
+import org.xmlpull.v1.XmlPullParser;
+import org.xmlpull.v1.XmlPullParserException;
+import org.xmlpull.v1.XmlPullParserFactory;
+
+import junit.framework.TestCase;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+
+/** Tests for {@link TestLog}. */
+public class TestLogTest extends TestCase {
+
+ public void testTestLogType_fromDataName() {
+ assertNull(TestLogType.fromDataName(null));
+ assertNull(TestLogType.fromDataName(""));
+ assertNull(TestLogType.fromDataName("kmsg-foo_bar_test"));
+
+ assertEquals(TestLogType.LOGCAT,
+ TestLogType.fromDataName("logcat-foo_bar_test"));
+ assertEquals(TestLogType.BUGREPORT,
+ TestLogType.fromDataName("bug-foo_bar_test"));
+ }
+
+ public void testTestLogType_getAttrValue() {
+ assertEquals("logcat", TestLogType.LOGCAT.getAttrValue());
+ assertEquals("bugreport", TestLogType.BUGREPORT.getAttrValue());
+ }
+
+ public void testFromDataName() {
+ TestLog log = TestLog.fromDataName("logcat-baz_test", "http://logs/baz_test");
+ assertEquals(TestLogType.LOGCAT, log.getLogType());
+ assertEquals("http://logs/baz_test", log.getUrl());
+ }
+
+ public void testFromDataName_unrecognizedDataName() {
+ assertNull(TestLog.fromDataName("kmsg-baz_test", null));
+ }
+
+ public void testFromDataName_nullDataName() {
+ assertNull(TestLog.fromDataName(null, "http://logs/baz_test"));
+ }
+
+ public void testFromDataName_nullUrl() {
+ assertNull(TestLog.fromDataName("logcat-bar_test", null));
+ }
+
+ public void testFromDataName_allNull() {
+ assertNull(TestLog.fromDataName(null, null));
+ }
+
+ public void testFromXml() throws Exception {
+ TestLog log = TestLog.fromXml(newXml("<TestLog type=\"logcat\" url=\"http://logs/baz_test\">"));
+ assertEquals(TestLogType.LOGCAT, log.getLogType());
+ assertEquals("http://logs/baz_test", log.getUrl());
+ }
+
+ public void testFromXml_unrecognizedType() throws Exception {
+ assertNull(TestLog.fromXml(newXml("<TestLog type=\"kmsg\" url=\"http://logs/baz_test\">")));
+ }
+
+ public void testFromXml_noTypeAttribute() throws Exception {
+ assertNull(TestLog.fromXml(newXml("<TestLog url=\"http://logs/baz_test\">")));
+ }
+
+ public void testFromXml_noUrlAttribute() throws Exception {
+ assertNull(TestLog.fromXml(newXml("<TestLog type=\"bugreport\">")));
+ }
+
+ public void testFromXml_allNull() throws Exception {
+ assertNull(TestLog.fromXml(newXml("<TestLog>")));
+ }
+
+ public void testSerialize() throws Exception {
+ KXmlSerializer serializer = new KXmlSerializer();
+ StringWriter writer = new StringWriter();
+ serializer.setOutput(writer);
+
+ TestLog log = TestLog.of(TestLogType.LOGCAT, "http://logs/foo/bar");
+ log.serialize(serializer);
+ assertEquals("<TestLog type=\"logcat\" url=\"http://logs/foo/bar\" />", writer.toString());
+ }
+
+ public void testIsTag() {
+ assertTrue(TestLog.isTag("TestLog"));
+ assertFalse(TestLog.isTag("TestResult"));
+ }
+
+ private XmlPullParser newXml(String xml) throws Exception {
+ XmlPullParserFactory factory = org.xmlpull.v1.XmlPullParserFactory.newInstance();
+ XmlPullParser parser = factory.newPullParser();
+ parser.setInput(new StringReader(xml));
+
+ // Move the parser from the START_DOCUMENT stage to the START_TAG of the data.
+ parser.next();
+
+ return parser;
+ }
+}
