change summary to average/stddev rather than average/worst
- stddev will tell the fluctuation in performance which can be better
indicator than worst which can be caused by unknown background task
- also add get_csv_report.py which generates csv report under a given dir.
Change-Id: I70333d968beaa159fa3cde15147ec59dd469efe3
diff --git a/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java b/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
index 9c23f22..28ae4a9 100644
--- a/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
+++ b/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
@@ -52,8 +52,9 @@
Log.i(TAG, " x " + size.x + " y " + size.y);
double pixels = size.x * size.y;
// now this represents how many times the whole screen can be copied in a sec.
- double screensPerSecMin = stat.mMin / pixels * 1024.0 * 1024.0 / 4.0;
double screensPerSecAverage = stat.mAverage / pixels * 1024.0 * 1024.0 / 4.0;
- getReportLog().printSummary("screen copies per sec", screensPerSecMin, screensPerSecAverage);
+ double screensPerSecStddev = stat.mStddev / pixels * 1024.0 * 1024.0 / 4.0;
+ getReportLog().printSummary("screen copies per sec", screensPerSecAverage,
+ screensPerSecStddev);
}
}
diff --git a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FileUtil.java b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FileUtil.java
index 57f868a..2cb3854 100644
--- a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FileUtil.java
+++ b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FileUtil.java
@@ -276,7 +276,6 @@
final int runsInOneGo = 16;
final int readsInOneMeasure = totalReadCount / runsInOneGo;
-
final RandomAccessFile randomFile = new RandomAccessFile(file, "rw");
double[] rdAmount = new double[runsInOneGo];
double[] wrAmount = new double[runsInOneGo];
@@ -300,7 +299,7 @@
report.printArray("Rd amount", rdAmount, true);
Stat.StatResult stat = Stat.getStat(mbps);
- report.printSummary("MB/s", stat.mMin, stat.mAverage);
+ report.printSummary("MB/s", stat.mAverage, stat.mStddev);
}
/**
@@ -327,7 +326,6 @@
final int runsInOneGo = 16;
final int writesInOneMeasure = totalWriteCount / runsInOneGo; // 32MB at a time
-
final RandomAccessFile randomFile = new RandomAccessFile(file, "rw");
double[] rdAmount = new double[runsInOneGo];
double[] wrAmount = new double[runsInOneGo];
@@ -351,7 +349,7 @@
report.printArray("Wr amount", wrAmount, true);
Stat.StatResult stat = Stat.getStat(mbps);
- report.printSummary("MB/s", stat.mMin, stat.mAverage);
+ report.printSummary("MB/s", stat.mAverage, stat.mStddev);
}
/**
@@ -369,12 +367,11 @@
File file = FileUtil.createNewFilledFile(context,
dirName, fileSize);
final byte[] data = FileUtil.generateRandomData(bufferSize);
- double[] worsts = new double[numberRepetition];
- double[] averages = new double[numberRepetition];
+ int numberRepeatInOneRun = (int)(fileSize / bufferSize);
+ double[] mbpsAll = new double[numberRepetition * numberRepeatInOneRun];
for (int i = 0; i < numberRepetition; i++) {
final FileOutputStream out = new FileOutputStream(file);
- int numberRepeat = (int)(fileSize / bufferSize);
- double[] times = MeasureTime.measure(numberRepeat, new MeasureRun() {
+ double[] times = MeasureTime.measure(numberRepeatInOneRun, new MeasureRun() {
@Override
public void run(int i) throws IOException {
@@ -387,12 +384,9 @@
times);
report.printArray(i + "-th round MB/s",
mbps, true);
- Stat.StatResult stat = Stat.getStat(mbps);
- worsts[i] = stat.mMin;
- averages[i] = stat.mAverage;
+ ReportLog.copyArray(mbps, mbpsAll, i * numberRepeatInOneRun);
}
- Stat.StatResult statWorsts = Stat.getStat(worsts);
- Stat.StatResult statAverages = Stat.getStat(averages);
- report.printSummary("MB/s", statWorsts.mMin, statAverages.mAverage);
+ Stat.StatResult stat = Stat.getStat(mbpsAll);
+ report.printSummary("MB/s", stat.mAverage, stat.mStddev);
}
}
diff --git a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/SequentialRWTest.java b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/SequentialRWTest.java
index eb66da2..252c59e 100644
--- a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/SequentialRWTest.java
+++ b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/SequentialRWTest.java
@@ -65,7 +65,7 @@
mbps, true);
getReportLog().printArray("Wr amount", wrAmount, true);
Stat.StatResult stat = Stat.getStat(mbps);
- getReportLog().printSummary("MB/s", stat.mMin, stat.mAverage);
+ getReportLog().printSummary("MB/s", stat.mAverage, stat.mStddev);
}
@TimeoutReq(minutes = 60)
@@ -106,6 +106,6 @@
getReportLog().printArray("read MB/s",
mbps, true);
Stat.StatResult stat = Stat.getStat(mbps);
- getReportLog().printSummary("MB/s", stat.mMin, stat.mAverage);
+ getReportLog().printSummary("MB/s", stat.mAverage, stat.mStddev);
}
}
diff --git a/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java b/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
index 7491f8a..8e4bbed 100644
--- a/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
+++ b/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
@@ -67,6 +67,6 @@
});
getReportLog().printArray("ms", results, false);
Stat.StatResult stat = Stat.getStat(results);
- getReportLog().printSummary("Time ms", stat.mMax, stat.mAverage);
+ getReportLog().printSummary("Time ms", stat.mAverage, stat.mStddev);
}
}
diff --git a/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java b/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
index 97d9dc8..709588b 100644
--- a/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
+++ b/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
@@ -63,7 +63,7 @@
});
mReport.printArray("time in ms", result, false);
StatResult stat = Stat.getStat(result);
- mReport.printSummary("time in ms", stat.mMax, stat.mAverage);
+ mReport.printSummary("time in ms", stat.mAverage, stat.mStddev);
}
private void rebootDevice() throws DeviceNotAvailableException {
diff --git a/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java b/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
index 1fe3c59..0d69c2f 100644
--- a/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
+++ b/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
@@ -72,9 +72,9 @@
printLog(builder.toString());
}
- public void printSummary(String header, double worst, double average) {
- mSummary = header + LOG_ELEM_SEPARATOR + "worst " + worst + LOG_ELEM_SEPARATOR +
- "average " + average;
+ public void printSummary(String header, double average, double stddev) {
+ mSummary = header + LOG_ELEM_SEPARATOR + "average " + average + LOG_ELEM_SEPARATOR +
+ "stddev " + stddev;
}
public void throwReportToHost() throws PtsException {
@@ -132,6 +132,19 @@
}
/**
+ * copy array from src to dst with given offset in dst.
+ * dst should be big enough to hold src
+ * @param src
+ * @param dst
+ * @param dstOffset
+ */
+ public static void copyArray(double[] src, double[] dst, int dstOffset) {
+ for (int i = 0; i < src.length; i++) {
+ dst[dstOffset + i] = src[i];
+ }
+ }
+
+ /**
* get classname.methodname from call stack of the current thread
*
* @return
diff --git a/suite/pts/utils/get_csv_report.py b/suite/pts/utils/get_csv_report.py
new file mode 100755
index 0000000..fe3dd74
--- /dev/null
+++ b/suite/pts/utils/get_csv_report.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import csv
+import os
+import re
+import subprocess
+import sys
+from xml.dom import Node
+from xml.dom import minidom
+
+
+def getChildrenWithTag(parent, tagName):
+ children = []
+ for child in parent.childNodes:
+ if (child.nodeType == Node.ELEMENT_NODE) and (child.tagName == tagName):
+ #print "parent " + parent.getAttribute("name") + " " + tagName +\
+ # " " + child.getAttribute("name")
+ children.append(child)
+ return children
+
+class TestCase(object):
+ def __init__(self, name, average, stddev, passFail):
+ self.name = name
+ self.average = average
+ self.stddev = stddev
+ self.passFail = passFail
+
+ def getName(self):
+ return self.name
+
+ def getStddev(self):
+ return self.stddev
+
+ def getAverage(self):
+ return self.average
+
+ def getPassFail(self):
+ return self.passFail
+
+def parseSuite(suite, parentName):
+ if parentName != "":
+ parentName += '.'
+ cases = {}
+ childSuites = getChildrenWithTag(suite, "TestSuite")
+ for child in childSuites:
+ cases.update(parseSuite(child, parentName + child.getAttribute("name")))
+ childTestCases = getChildrenWithTag(suite, "TestCase")
+ for child in childTestCases:
+ className = parentName + child.getAttribute("name")
+ for test in getChildrenWithTag(child, "Test"):
+ methodName = test.getAttribute("name")
+ # do not include this
+ if methodName == "testAndroidTestCaseSetupProperly":
+ continue
+ caseName = className + "#" + methodName
+ passFail = test.getAttribute("result")
+ average = ""
+ stddev = ""
+ failedScene = getChildrenWithTag(test, "FailedScene")
+ if len(failedScene) > 0:
+ message = failedScene[0].getAttribute("message")
+ #print message
+ messages = message.split('|')
+ if len(messages) > 2:
+ average = messages[1].split()[1]
+ stddev = messages[2].split()[1]
+ testCase = TestCase(caseName, average, stddev, passFail)
+ cases[caseName] = testCase
+ return cases
+
+
+class Result(object):
+ def __init__(self, reportXml):
+ self.results = {}
+ self.infoKeys = []
+ self.infoValues = []
+ doc = minidom.parse(reportXml)
+ testResult = doc.getElementsByTagName("TestResult")[0]
+ buildInfo = testResult.getElementsByTagName("BuildInfo")[0]
+ buildId = buildInfo.getAttribute("buildID")
+ deviceId = buildInfo.getAttribute("deviceID")
+ deviceName = buildInfo.getAttribute("build_device")
+ boardName = buildInfo.getAttribute("build_board")
+ partitions = buildInfo.getAttribute("partitions")
+ m = re.search(r'.*;/data\s+(\w+)\s+(\w+)\s+(\w+)\s+(\w+);', partitions)
+ dataPartitionSize = m.group(1)
+ self.addKV("device", deviceName)
+ self.addKV("board", boardName)
+ self.addKV("serial", deviceId)
+ self.addKV("build", buildId)
+ self.addKV("data size", dataPartitionSize)
+ packages = getChildrenWithTag(testResult, "TestPackage")
+ for package in packages:
+ casesFromChild = parseSuite(package, "")
+ self.results.update(casesFromChild)
+ #print self.results.keys()
+
+ def addKV(self, key, value):
+ self.infoKeys.append(key)
+ self.infoValues.append(value)
+
+ def getResults(self):
+ return self.results
+
+ def getKeys(self):
+ return self.infoKeys
+
+ def getValues(self):
+ return self.infoValues
+
+def executeWithResult(command):
+ p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ out, err = p.communicate()
+ return out
+
+def main(argv):
+ if len(argv) < 3:
+ print "get_csv_report.py pts_report_dir output_file"
+ sys.exit(1)
+ reportPath = os.path.abspath(argv[1])
+ outputCsv = os.path.abspath(argv[2])
+
+ deviceResults = []
+ xmls = executeWithResult("find " + reportPath + " -name testResult.xml -print")
+ print "xml files found :"
+ print xmls
+ for xml in xmls.splitlines():
+ result = Result(xml)
+ deviceResults.append(result)
+ reportInfo = []
+ keys = deviceResults[0].getKeys()
+ noDevices = len(deviceResults)
+ for i in xrange(len(keys)):
+ reportInfo.append([])
+ reportInfo[i].append(keys[i])
+ # for worst/average
+ reportInfo[i].append("")
+ for j in xrange(noDevices):
+ reportInfo[i].append(deviceResults[j].getValues()[i])
+ #print reportInfo
+
+ tests = []
+ for deviceResult in deviceResults:
+ for key in deviceResult.getResults().keys():
+ if not key in tests:
+ tests.append(key)
+ tests.sort()
+ #print tests
+
+ reportTests = []
+ for i in xrange(len(tests)):
+ reportTests.append([])
+ reportTests.append([])
+ reportTests[2 * i].append(tests[i])
+ reportTests[2 * i + 1].append(tests[i])
+ reportTests[2 * i].append("average")
+ reportTests[2 * i + 1].append("stddev")
+ for j in xrange(noDevices):
+ if deviceResults[j].getResults().has_key(tests[i]):
+ result = deviceResults[j].getResults()[tests[i]]
+ if result.getPassFail() == "pass":
+ reportTests[2 * i].append(result.getAverage())
+ reportTests[2 * i + 1].append(result.getStddev())
+ else:
+ reportTests[2 * i].append("fail")
+ reportTests[2 * i + 1].append("fail")
+ else:
+ reportTests[2 * i].append("")
+ reportTests[2 * i + 1].append("")
+
+ #print reportTests
+
+ with open(outputCsv, 'wb') as f:
+ writer = csv.writer(f)
+ writer.writerows(reportInfo)
+ writer.writerows(reportTests)
+
+
+if __name__ == '__main__':
+ main(sys.argv)
