Enhance CTS-Verifier to record whether the test item is automated.
- Record whether is automated of each execution for a test item.
- Remove unused function and it's unit test(TestResultHistory.serialize)
Bug: 158059049
Test: Unit Test: cts-tradefed run cts-unit-tests
cts-tradefed run cts -m CtsMonkeyTestCases
Manual Test: Build cts-v and verify it works as intended
(including a new attribute "isAutomated" in existing tag <Run>)
Change-Id: Iaf51dd154a8037b7114267200986d33abb0eb27b
Merged-In: Iaf51dd154a8037b7114267200986d33abb0eb27b
diff --git a/common/util/src/com/android/compatibility/common/util/TestResultHistory.java b/common/util/src/com/android/compatibility/common/util/TestResultHistory.java
index 12f1da6..15a9319 100644
--- a/common/util/src/com/android/compatibility/common/util/TestResultHistory.java
+++ b/common/util/src/com/android/compatibility/common/util/TestResultHistory.java
@@ -16,14 +16,10 @@
package com.android.compatibility.common.util;
-import org.xmlpull.v1.XmlPullParserException;
-import org.xmlpull.v1.XmlPullParserFactory;
import org.xmlpull.v1.XmlSerializer;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.Serializable;
-import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -45,19 +41,20 @@
private static final String RUN_TAG = "Run";
private static final String START_TIME_ATTR = "start";
private static final String END_TIME_ATTR = "end";
+ private static final String IS_AUTOMATED_ATTR = "isAutomated";
private final String mTestName;
- private final Set<Map.Entry> mDurations;
+ private final Set<ExecutionRecord> mExecutionRecords;
/**
* Constructor of test result history.
*
* @param testName a string of test name.
- * @param durations a set of execution time.
+ * @param executionRecords a Set of ExecutionRecords.
*/
- public TestResultHistory(String testName, Set<Map.Entry> durations) {
+ public TestResultHistory(String testName, Set<ExecutionRecord> executionRecords) {
this.mTestName = testName;
- this.mDurations = durations;
+ this.mExecutionRecords = executionRecords;
}
/** Get test name */
@@ -65,9 +62,9 @@
return mTestName;
}
- /** Get a set of duration with key as start time and value as end time. */
- public Set<Map.Entry> getDurations() {
- return mDurations;
+ /** Get a set of ExecutionRecords. */
+ public Set<ExecutionRecord> getExecutionRecords() {
+ return mExecutionRecords;
}
/** {@inheritDoc} */
@@ -81,20 +78,20 @@
}
TestResultHistory that = (TestResultHistory) o;
return Objects.equals(mTestName, that.mTestName)
- && Objects.equals(mDurations, that.mDurations);
+ && Objects.equals(mExecutionRecords, that.mExecutionRecords);
}
/** {@inheritDoc} */
@Override
public int hashCode() {
- return Objects.hash(mTestName, mDurations);
+ return Objects.hash(mTestName, mExecutionRecords);
}
/**
* Serializes a given {@link TestResultHistory} to XML.
*
* @param serializer given serializer.
- * @param resultHistory test result history with test name and execution time.
+ * @param resultHistory test result history with test name and execution record.
* @param testName top-level test name.
* @throws IOException
*/
@@ -112,35 +109,63 @@
serializer.attribute(null, SUB_TEST_ATTR, name);
}
- for (Map.Entry<Long, Long> duration : resultHistory.getDurations()) {
+ for (ExecutionRecord execRecord : resultHistory.getExecutionRecords()) {
serializer.startTag(null, RUN_TAG);
- serializer.attribute(null, START_TIME_ATTR, String.valueOf(duration.getKey()));
- serializer.attribute(null, END_TIME_ATTR, String.valueOf(duration.getValue()));
+ serializer.attribute(null, START_TIME_ATTR, String.valueOf(execRecord.getStartTime()));
+ serializer.attribute(null, END_TIME_ATTR, String.valueOf(execRecord.getEndTime()));
+ serializer.attribute(
+ null, IS_AUTOMATED_ATTR, String.valueOf(execRecord.getIsAutomated()));
serializer.endTag(null, RUN_TAG);
}
serializer.endTag(null, RUN_HISTORY_TAG);
}
- /**
- * Serializes a given {@link TestResultHistory} to a String.
- *
- * @param resultHistory test result history with test name and execution time.
- * @param testName top-level test name.
- * @throws XmlPullParserException
- * @throws IOException
- * @throws IllegalStateException
- * @throws IllegalArgumentException
- */
- public static String serialize(TestResultHistory resultHistory, String testName)
- throws XmlPullParserException, IllegalArgumentException, IllegalStateException,
- IOException {
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- XmlSerializer serializer = XmlPullParserFactory.newInstance(TYPE, null).newSerializer();
- serializer.setOutput(byteArrayOutputStream, ENCODING);
- serializer.startDocument(ENCODING, true);
- serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
- serialize(serializer, resultHistory, testName);
- serializer.endDocument();
- return byteArrayOutputStream.toString(ENCODING);
+ /** Execution Record about start time, end time and isAutomated */
+ public static class ExecutionRecord implements Serializable {
+
+ private static final long serialVersionUID = 0L;
+ // Start time of test case.
+ private final long startTime;
+ // End time of test case.
+ private final long endTime;
+ // Whether test case was executed through automation.
+ private final boolean isAutomated;
+
+ public ExecutionRecord(long startTime, long endTime, boolean isAutomated) {
+ this.startTime = startTime;
+ this.endTime = endTime;
+ this.isAutomated = isAutomated;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getEndTime() {
+ return endTime;
+ }
+
+ public boolean getIsAutomated() {
+ return isAutomated;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ExecutionRecord that = (ExecutionRecord) o;
+ return startTime == that.startTime
+ && endTime == that.endTime
+ && isAutomated == that.isAutomated;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(startTime, endTime, isAutomated);
+ }
}
}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java b/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
index d7bcf2f..c60be6f 100644
--- a/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
+++ b/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
@@ -30,7 +30,6 @@
import java.io.StringReader;
import java.net.InetAddress;
import java.net.UnknownHostException;
-import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -107,9 +106,10 @@
private static final String END_DISPLAY = "Fri Aug 20 15:13:04 PDT 2010";
private static final long TEST_START_MS = 1000000000011L;
private static final long TEST_END_MS = 1000000000012L;
+ private static final boolean TEST_IS_AUTOMATED = false;
- private static final String REFERENCE_URL="http://android.com";
- private static final String LOG_URL ="file:///path/to/logs";
+ private static final String REFERENCE_URL = "http://android.com";
+ private static final String LOG_URL = "file:///path/to/logs";
private static final String COMMAND_LINE_ARGS = "cts -m CtsMyModuleTestCases";
private static final String XML_BASE =
"<?xml version='1.0' encoding='UTF-8' standalone='no' ?>" +
@@ -280,8 +280,12 @@
moduleB.setDone(true);
moduleB.addRuntime(Integer.parseInt(RUNTIME_B));
ICaseResult moduleBCase = moduleB.getOrCreateResult(CLASS_B);
- Set<Map.Entry> durations = new HashSet<>();
- durations.add(new AbstractMap.SimpleEntry<>(TEST_START_MS, TEST_END_MS));
+ Set<TestResultHistory.ExecutionRecord> executionRecords =
+ new HashSet<TestResultHistory.ExecutionRecord>();
+ executionRecords.add(
+ new TestResultHistory.ExecutionRecord(
+ TEST_START_MS, TEST_END_MS, TEST_IS_AUTOMATED));
+
ITestResult moduleBTest3 = moduleBCase.getOrCreateResult(METHOD_3);
moduleBTest3.setResultStatus(TestStatus.FAIL);
moduleBTest3.setMessage(MESSAGE);
@@ -290,7 +294,7 @@
moduleBTest3.setLog(LOGCAT);
moduleBTest3.setScreenshot(SCREENSHOT);
List<TestResultHistory> resultHistories = new ArrayList<TestResultHistory>();
- TestResultHistory resultHistory = new TestResultHistory(METHOD_3, durations);
+ TestResultHistory resultHistory = new TestResultHistory(METHOD_3, executionRecords);
resultHistories.add(resultHistory);
moduleBTest3.setTestResultHistories(resultHistories);
ITestResult moduleBTest4 = moduleBCase.getOrCreateResult(METHOD_4);
@@ -334,6 +338,11 @@
"Result/Module/TestCase/Test/RunHistory/Run",
"end",
Long.toString(TEST_END_MS));
+ assertXmlContainsAttribute(
+ content,
+ "Result/Module/TestCase/Test/RunHistory/Run",
+ "isAutomated",
+ Boolean.toString(TEST_IS_AUTOMATED));
checkResult(result, EXAMPLE_BUILD_FINGERPRINT, false);
}
@@ -532,9 +541,15 @@
for (TestResultHistory resultHistory : resultHistories) {
assertNotNull("Expected test result history", resultHistory);
assertEquals("Incorrect test name", METHOD_3, resultHistory.getTestName());
- for (Map.Entry duration : resultHistory.getDurations()) {
- assertEquals("Incorrect test start time", TEST_START_MS, duration.getKey());
- assertEquals("Incorrect test end time", TEST_END_MS, duration.getValue());
+ for (TestResultHistory.ExecutionRecord execRecord :
+ resultHistory.getExecutionRecords()) {
+ assertEquals(
+ "Incorrect test start time", TEST_START_MS, execRecord.getStartTime());
+ assertEquals("Incorrect test end time", TEST_END_MS, execRecord.getEndTime());
+ assertEquals(
+ "Incorrect test is automated",
+ TEST_IS_AUTOMATED,
+ execRecord.getIsAutomated());
}
}
}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/TestResultHistoryTest.java b/common/util/tests/src/com/android/compatibility/common/util/TestResultHistoryTest.java
deleted file mode 100644
index 740b084..0000000
--- a/common/util/tests/src/com/android/compatibility/common/util/TestResultHistoryTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright (C) 2019 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.compatibility.common.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-import java.util.AbstractMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/** Unit tests for {@link TestResultHistory} */
-@RunWith(JUnit4.class)
-public class TestResultHistoryTest {
-
- private static final String TEST_NAME = "Test";
- private static final long START_TIME = 1000000000011L;
- private static final long END_TIME = 1000000000012L;
- private static final String HEADER_XML =
- "<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>";
- private static final String RUN_HISTORIES_XML =
- HEADER_XML
- + "\r\n"
- + "<RunHistory>\r\n"
- + " <Run start=\"1000000000011\" end=\"1000000000012\" />\r\n"
- + "</RunHistory>";
-
- private TestResultHistory mTestResultHistory;
- private Set<Map.Entry> mDurations;
-
- @Before
- public void setUp() throws Exception {
- mDurations = new HashSet<>();
- }
-
- @Test
- public void testSerialize_null() throws Exception {
- try {
- TestResultHistory.serialize(null, null);
- fail("Expected IllegalArgumentException when serializing an empty test result history");
- } catch (IllegalArgumentException e) {
- // Expected
- }
- }
-
- @Test
- public void testSerialize_full() throws Exception {
- mDurations.add(new AbstractMap.SimpleEntry<>(START_TIME, END_TIME));
- mTestResultHistory = new TestResultHistory(TEST_NAME, mDurations);
- assertEquals(RUN_HISTORIES_XML, TestResultHistory.serialize(mTestResultHistory, TEST_NAME));
- }
-}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java b/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
index 60f07a7..948c26e 100644
--- a/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
+++ b/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
@@ -36,7 +36,6 @@
ReportLogTest.class,
ResultHandlerTest.class,
StatTest.class,
- TestResultHistoryTest.class,
TestResultTest.class,
ReadElfTest.class,
})
