reduce result fluctuation for memory and cpu tests
- introduce outlier rejection to ignore data too far from median values.
- also increase repetition for memset as memset shows wider fluctuation than memcpy
bug: 8775068
Change-Id: I78978040e747a182db3cebf311fcf776f2cadf4d
diff --git a/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java b/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
index 8f6860f..62b0163 100644
--- a/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
+++ b/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
@@ -35,11 +35,14 @@
*/
public class BandwidthTest extends PtsAndroidTestCase {
private static final String TAG = "BandwidthTest";
- private static final int REPETITION = 10;
+ private static final int MEMCPY_REPETITION = 10;
+ private static final int MEMSET_REPETITION = 30;
private static final int REPEAT_IN_EACH_CALL = 100;
private static final int KB = 1024;
private static final int MB = 1024 * 1024;
private static final int MEMSET_CHAR = 0xa5;
+ // reject data outside +/- this value * median
+ private static final double OUTLIER_THRESHOLD = 0.1;
@Override
protected void setUp() throws Exception {
@@ -153,13 +156,13 @@
}
private void doRunMemcpy(int bufferSize) {
- double[] result = new double[REPETITION];
+ double[] result = new double[MEMCPY_REPETITION];
int repeatInEachCall = REPEAT_IN_EACH_CALL;
if (bufferSize < (1 * MB)) {
// too small buffer size finishes too early to give accurate result.
repeatInEachCall *= (1 * MB / bufferSize);
}
- for (int i = 0; i < REPETITION; i++) {
+ for (int i = 0; i < MEMCPY_REPETITION; i++) {
result[i] = MemoryNative.runMemcpy(bufferSize, repeatInEachCall);
}
getReportLog().printArray("memcpy time", result, ResultType.LOWER_BETTER,
@@ -168,7 +171,10 @@
(double)bufferSize * repeatInEachCall / 1024.0 / 1024.0, result);
getReportLog().printArray("memcpy throughput", mbps, ResultType.HIGHER_BETTER,
ResultUnit.MBPS);
- Stat.StatResult stat = Stat.getStat(mbps);
+ Stat.StatResult stat = Stat.getStatWithOutlierRejection(mbps, OUTLIER_THRESHOLD);
+ if (stat.mDataCount != result.length) {
+ Log.w(TAG, "rejecting " + (result.length - stat.mDataCount) + " outliers");
+ }
WindowManager wm = (WindowManager) getContext().getSystemService(Context.WINDOW_SERVICE);
Point size = new Point();
wm.getDefaultDisplay().getSize(size);
@@ -183,13 +189,13 @@
}
private void doRunMemset(int bufferSize) {
- double[] result = new double[REPETITION];
+ double[] result = new double[MEMSET_REPETITION];
int repeatInEachCall = REPEAT_IN_EACH_CALL;
if (bufferSize < (1 * MB)) {
// too small buffer size finishes too early to give accurate result.
repeatInEachCall *= (1 * MB / bufferSize);
}
- for (int i = 0; i < REPETITION; i++) {
+ for (int i = 0; i < MEMSET_REPETITION; i++) {
result[i] = MemoryNative.runMemset(bufferSize, repeatInEachCall, MEMSET_CHAR);
}
getReportLog().printArray("memset time", result, ResultType.LOWER_BETTER,
@@ -198,7 +204,10 @@
(double)bufferSize * repeatInEachCall / 1024.0 / 1024.0, result);
getReportLog().printArray("memset throughput", mbps, ResultType.HIGHER_BETTER,
ResultUnit.MBPS);
- Stat.StatResult stat = Stat.getStat(mbps);
+ Stat.StatResult stat = Stat.getStatWithOutlierRejection(mbps, OUTLIER_THRESHOLD);
+ if (stat.mDataCount != result.length) {
+ Log.w(TAG, "rejecting " + (result.length - stat.mDataCount) + " outliers");
+ }
WindowManager wm = (WindowManager) getContext().getSystemService(Context.WINDOW_SERVICE);
Point size = new Point();
wm.getDefaultDisplay().getSize(size);
diff --git a/suite/pts/deviceTests/simplecpu/src/com/android/pts/simplecpu/SimpleCpuTest.java b/suite/pts/deviceTests/simplecpu/src/com/android/pts/simplecpu/SimpleCpuTest.java
index 8b17171..8278618 100644
--- a/suite/pts/deviceTests/simplecpu/src/com/android/pts/simplecpu/SimpleCpuTest.java
+++ b/suite/pts/deviceTests/simplecpu/src/com/android/pts/simplecpu/SimpleCpuTest.java
@@ -17,6 +17,7 @@
package com.android.pts.simplecpu;
import android.cts.util.TimeoutReq;
+import android.util.Log;
import com.android.pts.util.ResultType;
import com.android.pts.util.ResultUnit;
@@ -35,6 +36,8 @@
private static final int KB = 1024;
private static final int MB = 1024 * 1024;
private static final int NUMBER_REPEAT = 20;
+ // reject data outside +/- this value * median
+ private static final double OUTLIER_THRESHOLD = 0.1;
@Override
protected void setUp() throws Exception {
@@ -99,7 +102,10 @@
}
getReportLog().printArray("sorting time", result, ResultType.LOWER_BETTER,
ResultUnit.MS);
- Stat.StatResult stat = Stat.getStat(result);
+ Stat.StatResult stat = Stat.getStatWithOutlierRejection(result, OUTLIER_THRESHOLD);
+ if (stat.mDataCount != result.length) {
+ Log.w(TAG, "rejecting " + (result.length - stat.mDataCount) + " outliers");
+ }
getReportLog().printSummary("sorting time", stat.mAverage, ResultType.LOWER_BETTER,
ResultUnit.MS);
}
@@ -118,7 +124,10 @@
}
getReportLog().printArray("matrix mutiplication time", result, ResultType.LOWER_BETTER,
ResultUnit.MS);
- Stat.StatResult stat = Stat.getStat(result);
+ Stat.StatResult stat = Stat.getStatWithOutlierRejection(result, OUTLIER_THRESHOLD);
+ if (stat.mDataCount != result.length) {
+ Log.w(TAG, "rejecting " + (result.length - stat.mDataCount) + " outliers");
+ }
getReportLog().printSummary("matrix mutiplication time", stat.mAverage,
ResultType.LOWER_BETTER, ResultUnit.MS);
}
diff --git a/suite/pts/lib/commonutil/src/com/android/pts/util/Stat.java b/suite/pts/lib/commonutil/src/com/android/pts/util/Stat.java
index d56f59e..5560292 100644
--- a/suite/pts/lib/commonutil/src/com/android/pts/util/Stat.java
+++ b/suite/pts/lib/commonutil/src/com/android/pts/util/Stat.java
@@ -16,6 +16,8 @@
package com.android.pts.util;
+import java.util.Arrays;
+
/**
* Utilities for doing statistics
*
@@ -30,11 +32,13 @@
public double mMin;
public double mMax;
public double mStddev;
- public StatResult(double average, double min, double max, double stddev) {
+ public int mDataCount;
+ public StatResult(double average, double min, double max, double stddev, int dataCount) {
mAverage = average;
mMin = min;
mMax = max;
mStddev = stddev;
+ mDataCount = dataCount;
}
}
@@ -60,7 +64,58 @@
eX2 /= data.length;
// stddev = sqrt(E[X^2] - (E[X])^2)
double stddev = Math.sqrt(eX2 - average * average);
- return new StatResult(average, min, max, stddev);
+ return new StatResult(average, min, max, stddev, data.length);
+ }
+
+ /**
+ * Calculate statistics properties likes average, min, max, and stddev for the given array
+ * while rejecting outlier +/- median * rejectionThreshold.
+ * rejectionThreshold should be bigger than 0.0 and be lowerthan 1.0
+ */
+ public static StatResult getStatWithOutlierRejection(double[] data, double rejectionThreshold) {
+ double[] dataCopied = Arrays.copyOf(data, data.length);
+ Arrays.sort(dataCopied);
+ int medianIndex = dataCopied.length / 2;
+ double median;
+ if (dataCopied.length % 2 == 1) {
+ median = dataCopied[medianIndex];
+ } else {
+ median = (dataCopied[medianIndex - 1] + dataCopied[medianIndex]) / 2.0;
+ }
+ double thresholdMin = median * (1.0 - rejectionThreshold);
+ double thresholdMax = median * (1.0 + rejectionThreshold);
+
+ double average = 0.0;
+ double min = median;
+ double max = median;
+ double eX2 = 0.0; // will become E[X^2]
+ int validDataCounter = 0;
+ for (int i = 0; i < data.length; i++) {
+ if ((data[i] > thresholdMin) && (data[i] < thresholdMax)) {
+ validDataCounter++;
+ average += data[i];
+ eX2 += data[i] * data[i];
+ if (data[i] > max) {
+ max = data[i];
+ }
+ if (data[i] < min) {
+ min = data[i];
+ }
+ }
+ //TODO report rejected data
+ }
+ double stddev;
+ if (validDataCounter > 0) {
+ average /= validDataCounter;
+ eX2 /= validDataCounter;
+ // stddev = sqrt(E[X^2] - (E[X])^2)
+ stddev = Math.sqrt(eX2 - average * average);
+ } else { // both median is showing too much diff
+ average = median;
+ stddev = 0; // don't care
+ }
+
+ return new StatResult(average, min, max, stddev, validDataCounter);
}
/**
