Add additional PTS code to common/cts
-Add ReportLog string encode/decode to common PTS package
-Add MetricsStore for host side metrics collection
bug:19901050
Change-Id: I95a17c8401096e1602c889894550fb97cd11bbcd
diff --git a/common/util/src/com/android/compatibility/common/util/MetricsReportLog.java b/common/util/src/com/android/compatibility/common/util/MetricsReportLog.java
new file mode 100644
index 0000000..4675231
--- /dev/null
+++ b/common/util/src/com/android/compatibility/common/util/MetricsReportLog.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package com.android.compatibility.common.util;
+
+/**
+ * A {@link ReportLog} that can be used with the in memory metrics store used for host side metrics.
+ */
+public final class MetricsReportLog extends ReportLog {
+ private final String mDeviceSerial;
+ private final String mAbi;
+ private final String mClassMethodName;
+
+ /**
+ * @param deviceSerial serial number of the device
+ * @param abi abi the test was run on
+ * @param classMethodName class name and method name of the test in class#method format.
+ * Note that ReportLog.getClassMethodNames() provide this.
+ */
+ public MetricsReportLog(String deviceSerial, String abi, String classMethodName) {
+ mDeviceSerial = deviceSerial;
+ mAbi = abi;
+ mClassMethodName = classMethodName;
+ }
+
+ public void submit() {
+ MetricsStore.storeResult(mDeviceSerial, mAbi, mClassMethodName, this);
+ }
+}
diff --git a/common/util/src/com/android/compatibility/common/util/MetricsStore.java b/common/util/src/com/android/compatibility/common/util/MetricsStore.java
new file mode 100644
index 0000000..9eeb94a
--- /dev/null
+++ b/common/util/src/com/android/compatibility/common/util/MetricsStore.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package com.android.compatibility.common.util;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A simple in-memory store for metrics results. This should be used for hostside metrics reporting.
+ */
+public class MetricsStore {
+
+ // needs concurrent version as there can be multiple client accessing this.
+ // But there is no additional protection for the same key as that should not happen.
+ private static final ConcurrentHashMap<String, ReportLog> mMap =
+ new ConcurrentHashMap<String, ReportLog>();
+
+ /**
+ * Stores a result. Existing result with the same key will be replaced.
+ * Note that key is generated in the form of device_serial#class#method name.
+ * So there should be no concurrent test for the same (serial, class, method).
+ * @param deviceSerial
+ * @param abi
+ * @param classMethodName
+ * @param reportLog Contains the result to be stored
+ */
+ public static void storeResult(
+ String deviceSerial, String abi, String classMethodName, ReportLog reportLog) {
+ mMap.put(generateTestKey(deviceSerial, abi, classMethodName), reportLog);
+ }
+
+ /**
+ * retrieves a metric result for the given condition and remove it from the internal
+ * storage. If there is no result for the given condition, it will return null.
+ */
+ public static ReportLog removeResult(String deviceSerial, String abi, String classMethodName) {
+ return mMap.remove(generateTestKey(deviceSerial, abi, classMethodName));
+ }
+
+ /**
+ * @return test key in the form of device_serial#abi#class_name#method_name
+ */
+ private static String generateTestKey(String deviceSerial, String abi, String classMethodName) {
+ return String.format("%s#%s#%s", deviceSerial, abi, classMethodName);
+ }
+}
diff --git a/common/util/src/com/android/compatibility/common/util/ReportLog.java b/common/util/src/com/android/compatibility/common/util/ReportLog.java
index 8cfc086..7209ac8 100644
--- a/common/util/src/com/android/compatibility/common/util/ReportLog.java
+++ b/common/util/src/com/android/compatibility/common/util/ReportLog.java
@@ -16,28 +16,37 @@
package com.android.compatibility.common.util;
-import org.xmlpull.v1.XmlSerializer;
-
-import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
+import java.util.StringTokenizer;
+import java.util.regex.Pattern;
/**
* Utility class to add results to the report.
*/
-public abstract class ReportLog implements Serializable {
+public class ReportLog implements Serializable {
+ private static final String LOG_SEPARATOR = "+++";
+ private static final String SUMMARY_SEPARATOR = "++++";
+ private static final String LOG_ELEM_SEPARATOR = "|";
+ private static final String EMPTY_CHAR = " ";
private Result mSummary;
private final List<Result> mDetails = new ArrayList<Result>();
- class Result implements Serializable {
- private static final int CALLER_STACKTRACE_DEPTH = 5;
+ static class Result implements Serializable {
private String mLocation;
private String mMessage;
private double[] mValues;
private ResultType mType;
private ResultUnit mUnit;
+ private Double mTarget;
+
+
+ private Result(String location, String message, double[] values,
+ ResultType type, ResultUnit unit) {
+ this(location, message, values, null /*target*/, type, unit);
+ }
/**
* Creates a result object to be included in the report. Each object has a message
@@ -47,23 +56,22 @@
*
* @param message A string describing the values
* @param values An array of the values
+ * @param target Nullable. The target value.
* @param type Represents how to interpret the values (eg. A lower score is better)
* @param unit Represents the unit in which the values are (eg. Milliseconds)
- * @param depth A number used to increase the depth the stack is queried. This should only
- * be given in the case that the report is populated by a helper function, in which case it
- * would be 1, or else 0.
*/
- private Result(String message, double[] values, ResultType type,
- ResultUnit unit, int depth) {
- final StackTraceElement[] trace = Thread.currentThread().getStackTrace();
- final StackTraceElement e =
- trace[Math.min(CALLER_STACKTRACE_DEPTH + depth, trace.length - 1)];
- mLocation = String.format(
- "%s#%s:%d", e.getClassName(), e.getMethodName(), e.getLineNumber());
+ private Result(String location, String message, double[] values,
+ Double target, ResultType type, ResultUnit unit) {
+ mLocation = location;
mMessage = message;
mValues = values;
mType = type;
mUnit = unit;
+ mTarget = target;
+ }
+
+ public double getTarget() {
+ return mTarget;
}
public String getLocation() {
@@ -85,51 +93,89 @@
public ResultUnit getUnit() {
return mUnit;
}
+
+ /**
+ * Format:
+ * location|message|target|type|unit|value[s], target can be " " if there is no target set.
+ * log for array = classMethodName:line_number|message|unit|type|space separated values
+ */
+ String toEncodedString() {
+ StringBuilder builder = new StringBuilder()
+ .append(mLocation)
+ .append(LOG_ELEM_SEPARATOR)
+ .append(mMessage)
+ .append(LOG_ELEM_SEPARATOR)
+ .append(mTarget != null ? mTarget : EMPTY_CHAR)
+ .append(LOG_ELEM_SEPARATOR)
+ .append(mType.name())
+ .append(LOG_ELEM_SEPARATOR)
+ .append(mUnit.name())
+ .append(LOG_ELEM_SEPARATOR);
+ for (double value : mValues) {
+ builder.append(value).append(" ");
+ }
+ return builder.toString();
+ }
+
+ static Result fromEncodedString(String encodedString) {
+ String[] elems = encodedString.split(Pattern.quote(LOG_ELEM_SEPARATOR));
+ if (elems.length < 5) {
+ return null;
+ }
+
+ String[] valueStrArray = elems[5].split(" ");
+ double[] valueArray = new double[valueStrArray.length];
+ for (int i = 0; i < valueStrArray.length; i++) {
+ valueArray[i] = Double.parseDouble(valueStrArray[i]);
+ }
+ return new Result(
+ elems[0], /*location*/
+ elems[1], /*message*/
+ valueArray, /*values*/
+ elems[2].equals(EMPTY_CHAR) ? null : Double.parseDouble(elems[2]), /*target*/
+ ResultType.valueOf(elems[3]), /*type*/
+ ResultUnit.valueOf(elems[4]) /*unit*/);
+ }
}
/**
* Adds an array of values to the report.
*/
public void addValues(String message, double[] values, ResultType type, ResultUnit unit) {
- mDetails.add(new Result(message, values, type, unit, 0));
+ mDetails.add(new Result(Stacktrace.getTestCallerClassMethodNameLineNumber(),
+ message, values, type, unit));
}
/**
* Adds an array of values to the report.
*/
- public void addValues(String message, double[] values, ResultType type,
- ResultUnit unit, int depth) {
- mDetails.add(new Result(message, values, type, unit, depth));
+ public void addValues(
+ String message, double[] values, ResultType type, ResultUnit unit, String location) {
+ mDetails.add(new Result(location, message, values, type, unit));
}
/**
* Adds a value to the report.
*/
public void addValue(String message, double value, ResultType type, ResultUnit unit) {
- mDetails.add(new Result(message, new double[] {value}, type, unit, 0));
+ mDetails.add(new Result(Stacktrace.getTestCallerClassMethodNameLineNumber(), message,
+ new double[] {value}, type, unit));
}
/**
* Adds a value to the report.
*/
public void addValue(String message, double value, ResultType type,
- ResultUnit unit, int depth) {
- mDetails.add(new Result(message, new double[] {value}, type, unit, depth));
+ ResultUnit unit, String location) {
+ mDetails.add(new Result(location, message, new double[] {value}, type, unit));
}
/**
* Sets the summary of the report.
*/
public void setSummary(String message, double value, ResultType type, ResultUnit unit) {
- mSummary = new Result(message, new double[] {value}, type, unit, 0);
- }
-
- /**
- * Sets the summary of the report.
- */
- public void setSummary(String message, double value, ResultType type,
- ResultUnit unit, int depth) {
- mSummary = new Result(message, new double[] {value}, type, unit, depth);
+ mSummary = new Result(Stacktrace.getTestCallerClassMethodNameLineNumber(),
+ message, new double[] {value}, type, unit);
}
public Result getSummary() {
@@ -139,4 +185,46 @@
public List<Result> getDetailedMetrics() {
return new ArrayList<Result>(mDetails);
}
+
+ /**
+ * Parse a String encoded {@link com.android.compatibility.common.util.ReportLog}
+ */
+ public static ReportLog fromEncodedString(String encodedString) {
+ ReportLog reportLog = new ReportLog();
+ StringTokenizer tok = new StringTokenizer(encodedString, SUMMARY_SEPARATOR);
+ if (tok.hasMoreTokens()) {
+ // Extract the summary
+ reportLog.mSummary = Result.fromEncodedString(tok.nextToken());
+ }
+ if (tok.hasMoreTokens()) {
+ // Extract the detailed results
+ StringTokenizer detailedTok = new StringTokenizer(tok.nextToken(), LOG_SEPARATOR);
+ while (detailedTok.hasMoreTokens()) {
+ reportLog.mDetails.add(Result.fromEncodedString(detailedTok.nextToken()));
+ }
+ }
+ return reportLog;
+ }
+
+ /**
+ * @return a String representation of this report or null if not collected
+ */
+ protected String toEncodedString() {
+ if ((mSummary == null) && mDetails.isEmpty()) {
+ // just return empty string
+ return null;
+ }
+ StringBuilder builder = new StringBuilder();
+ builder.append(mSummary.toEncodedString());
+ builder.append(SUMMARY_SEPARATOR);
+ for (Result result : mDetails) {
+ builder.append(result.toEncodedString());
+ builder.append(LOG_SEPARATOR);
+ }
+ // delete the last separator
+ if (builder.length() >= LOG_SEPARATOR.length()) {
+ builder.delete(builder.length() - LOG_SEPARATOR.length(), builder.length());
+ }
+ return builder.toString();
+ }
}
diff --git a/common/util/src/com/android/compatibility/common/util/Stacktrace.java b/common/util/src/com/android/compatibility/common/util/Stacktrace.java
new file mode 100644
index 0000000..27bf9ca
--- /dev/null
+++ b/common/util/src/com/android/compatibility/common/util/Stacktrace.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package com.android.compatibility.common.util;
+
+/**
+ * Helper methods for dealing with stack traces
+ */
+public class Stacktrace {
+
+ private static final int SAFETY_DEPTH = 4;
+ private static final String TEST_POSTFIX = "Test";
+
+ private Stacktrace() {}
+
+ /**
+ * @return classname#methodname from call stack of the current thread
+ */
+ public static String getTestCallerClassMethodName() {
+ return getTestCallerClassMethodName(false /*includeLineNumber*/);
+ }
+
+ /**
+ * @return classname#methodname from call stack of the current thread
+ */
+ public static String getTestCallerClassMethodNameLineNumber() {
+ return getTestCallerClassMethodName(true /*includeLineNumber*/);
+ }
+
+ /**
+ * @return classname#methodname from call stack of the current thread
+ */
+ private static String getTestCallerClassMethodName(boolean includeLineNumber) {
+ StackTraceElement[] elements = Thread.currentThread().getStackTrace();
+ // Look for the first class name in the elements array that ends with Test
+ for (int i = 0; i < elements.length; i++) {
+ if (elements[i].getClassName().endsWith(TEST_POSTFIX)) {
+ return buildClassMethodName(elements, i, includeLineNumber);
+ }
+ }
+
+ // Use a reasonable default if the test name isn't found
+ return buildClassMethodName(elements, SAFETY_DEPTH, includeLineNumber);
+ }
+
+ private static String buildClassMethodName(
+ StackTraceElement[] elements, int depth, boolean includeLineNumber) {
+ depth = Math.min(depth, elements.length - 1);
+ StringBuilder builder = new StringBuilder();
+ builder.append(elements[depth].getClassName()).append("#")
+ .append(elements[depth].getMethodName());
+ if (includeLineNumber) {
+ builder.append(":").append(elements[depth].getLineNumber());
+ }
+ return builder.toString();
+ }
+}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/MetricsStoreTest.java b/common/util/tests/src/com/android/compatibility/common/util/MetricsStoreTest.java
new file mode 100644
index 0000000..944cc43
--- /dev/null
+++ b/common/util/tests/src/com/android/compatibility/common/util/MetricsStoreTest.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package com.android.compatibility.common.util;
+
+import junit.framework.TestCase;
+
+/**
+ * Unit tests for {@link MetricsStore}
+ */
+public class MetricsStoreTest extends TestCase {
+
+ private static final String DEVICE_SERIAL = "DEVICE_SERIAL";
+ private static final String ABI = "ABI";
+ private static final String CLASSMETHOD_NAME = "CLASSMETHOD_NAME";
+
+ private static final double[] VALUES = new double[] {1, 11, 21, 1211, 111221};
+
+ private ReportLog mReportLog;
+
+ @Override
+ protected void setUp() throws Exception {
+ this.mReportLog = new ReportLog();
+ }
+
+ public void testStoreAndRemove() {
+ mReportLog.setSummary("Sample Summary", 1.0, ResultType.HIGHER_BETTER, ResultUnit.BYTE);
+ mReportLog.addValues("Details", VALUES, ResultType.NEUTRAL, ResultUnit.FPS);
+ MetricsStore.storeResult(DEVICE_SERIAL, ABI, CLASSMETHOD_NAME, mReportLog);
+
+ ReportLog reportLog = MetricsStore.removeResult(DEVICE_SERIAL, ABI, CLASSMETHOD_NAME);
+ assertSame(mReportLog, reportLog);
+ assertNull(MetricsStore.removeResult("blah", ABI, CLASSMETHOD_NAME));
+ }
+
+}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/MetricsXmlSerializerTest.java b/common/util/tests/src/com/android/compatibility/common/util/MetricsXmlSerializerTest.java
index 70da820..05e69d8 100644
--- a/common/util/tests/src/com/android/compatibility/common/util/MetricsXmlSerializerTest.java
+++ b/common/util/tests/src/com/android/compatibility/common/util/MetricsXmlSerializerTest.java
@@ -36,7 +36,8 @@
HEADER
+ "<Summary message=\"Sample\" scoreType=\"higher_better\" unit=\"byte\">1.0</Summary>"
+ "<Details>"
- + "<ValueArray source=\"sun.reflect.NativeMethodAccessorImpl#invoke0:-2\""
+ + "<ValueArray source=\"com.android.compatibility.common.util."
+ + "MetricsXmlSerializerTest#testSerialize:84\""
+ " message=\"Details\" scoreType=\"neutral\" unit=\"fps\">"
+ "<Value>1.0</Value>"
+ "<Value>11.0</Value>"
diff --git a/common/util/tests/src/com/android/compatibility/common/util/ReportLogTest.java b/common/util/tests/src/com/android/compatibility/common/util/ReportLogTest.java
new file mode 100644
index 0000000..a5f3306
--- /dev/null
+++ b/common/util/tests/src/com/android/compatibility/common/util/ReportLogTest.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+
+package com.android.compatibility.common.util;
+
+import junit.framework.TestCase;
+
+import java.util.Arrays;
+
+/**
+ * Unit tests for {@link ReportLog}
+ */
+public class ReportLogTest extends TestCase {
+
+ private static final double[] VALUES = new double[] {1, 11, 21, 1211, 111221};
+
+ private static final String EXPECTED_ENCODED_REPORT_LOG =
+ "com.android.compatibility.common.util.ReportLogTest#testEncodeDecode:44|" +
+ "Sample Summary| |HIGHER_BETTER|BYTE|1.0 ++++" +
+ "com.android.compatibility.common.util.ReportLogTest#testEncodeDecode:45|" +
+ "Details| |NEUTRAL|FPS|1.0 11.0 21.0 1211.0 111221.0 ";
+ private ReportLog reportLog;
+
+ @Override
+ protected void setUp() throws Exception {
+ this.reportLog = new ReportLog();
+ }
+
+ public void testEncodeDecode() {
+
+ reportLog.setSummary("Sample Summary", 1.0, ResultType.HIGHER_BETTER, ResultUnit.BYTE);
+ reportLog.addValues("Details", VALUES, ResultType.NEUTRAL, ResultUnit.FPS);
+
+ String encodedReportLog = reportLog.toEncodedString();
+ assertEquals(EXPECTED_ENCODED_REPORT_LOG, encodedReportLog);
+
+ ReportLog decodedReportLog = ReportLog.fromEncodedString(encodedReportLog);
+ ReportLog.Result summary = reportLog.getSummary();
+ assertEquals("Sample Summary", summary.getMessage());
+ assertFalse(summary.getLocation().isEmpty());
+ assertEquals(ResultType.HIGHER_BETTER, summary.getType());
+ assertEquals(ResultUnit.BYTE, summary.getUnit());
+ assertTrue(Arrays.equals(new double[] {1.0}, summary.getValues()));
+
+ assertEquals(1, decodedReportLog.getDetailedMetrics().size());
+ ReportLog.Result detail = decodedReportLog.getDetailedMetrics().get(0);
+ assertEquals("Details", detail.getMessage());
+ assertFalse(detail.getLocation().isEmpty());
+ assertEquals(ResultType.NEUTRAL, detail.getType());
+ assertEquals(ResultUnit.FPS, detail.getUnit());
+ assertTrue(Arrays.equals(VALUES, detail.getValues()));
+
+ assertEquals(encodedReportLog, decodedReportLog.toEncodedString());
+ }
+}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java b/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
index b9a17e1..348c680 100644
--- a/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
+++ b/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
@@ -26,6 +26,8 @@
public UnitTests() {
super();
+ addTestSuite(MetricsStoreTest.class);
addTestSuite(MetricsXmlSerializerTest.class);
+ addTestSuite(ReportLogTest.class);
}
}
