add PerfResultType to the summary report
- Each test should tell if the result is better with higher score or lower score.
- Without this, backend needs to remember which way is better for each test.
- This information should be displayed in visualization / table as well so
that users do not need to remember which way is better.
Change-Id: I1ae5308d609887b2e0cc9a59c00667f892e29adb
diff --git a/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java b/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
index c731cc1..d675edc 100644
--- a/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
+++ b/suite/pts/deviceTests/dram/src/com/android/pts/dram/BandwidthTest.java
@@ -21,6 +21,7 @@
import android.util.Log;
import android.view.WindowManager;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.PtsAndroidTestCase;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
@@ -109,6 +110,6 @@
double screensPerSecAverage = stat.mAverage / pixels * 1024.0 * 1024.0 / 4.0;
double screensPerSecStddev = stat.mStddev / pixels * 1024.0 * 1024.0 / 4.0;
getReportLog().printSummary("screen copies per sec", screensPerSecAverage,
- screensPerSecStddev);
+ PerfResultType.HIGHER_BETTER, screensPerSecStddev);
}
}
diff --git a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FileUtil.java b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FileUtil.java
index 14fc45a..1e39441 100644
--- a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FileUtil.java
+++ b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/FileUtil.java
@@ -28,6 +28,7 @@
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
import com.android.pts.util.SystemUtil;
@@ -303,7 +304,7 @@
report.printArray("Rd amount", rdAmount, true);
Stat.StatResult stat = Stat.getStat(mbps);
- report.printSummary("MB/s", stat.mAverage, stat.mStddev);
+ report.printSummary("MB/s", stat.mAverage, PerfResultType.HIGHER_BETTER, stat.mStddev);
}
/**
@@ -353,7 +354,7 @@
report.printArray("Wr amount", wrAmount, true);
Stat.StatResult stat = Stat.getStat(mbps);
- report.printSummary("MB/s", stat.mAverage, stat.mStddev);
+ report.printSummary("MB/s", stat.mAverage, PerfResultType.HIGHER_BETTER, stat.mStddev);
}
/**
@@ -391,6 +392,6 @@
ReportLog.copyArray(mbps, mbpsAll, i * numberRepeatInOneRun);
}
Stat.StatResult stat = Stat.getStat(mbpsAll);
- report.printSummary("MB/s", stat.mAverage, stat.mStddev);
+ report.printSummary("MB/s", stat.mAverage, PerfResultType.HIGHER_BETTER, stat.mStddev);
}
}
diff --git a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/SequentialRWTest.java b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/SequentialRWTest.java
index 252c59e..52dcb16 100644
--- a/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/SequentialRWTest.java
+++ b/suite/pts/deviceTests/filesystemperf/src/com/android/pts/filesystemperf/SequentialRWTest.java
@@ -19,6 +19,7 @@
import android.cts.util.TimeoutReq;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.PtsAndroidTestCase;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
@@ -65,7 +66,8 @@
mbps, true);
getReportLog().printArray("Wr amount", wrAmount, true);
Stat.StatResult stat = Stat.getStat(mbps);
- getReportLog().printSummary("MB/s", stat.mAverage, stat.mStddev);
+ getReportLog().printSummary("MB/s", stat.mAverage, PerfResultType.HIGHER_BETTER,
+ stat.mStddev);
}
@TimeoutReq(minutes = 60)
@@ -106,6 +108,7 @@
getReportLog().printArray("read MB/s",
mbps, true);
Stat.StatResult stat = Stat.getStat(mbps);
- getReportLog().printSummary("MB/s", stat.mAverage, stat.mStddev);
+ getReportLog().printSummary("MB/s", stat.mAverage, PerfResultType.HIGHER_BETTER,
+ stat.mStddev);
}
}
diff --git a/suite/pts/deviceTests/simplecpu/src/com/android/pts/simplecpu/SimpleCpuTest.java b/suite/pts/deviceTests/simplecpu/src/com/android/pts/simplecpu/SimpleCpuTest.java
index df8cfd9..4231250 100644
--- a/suite/pts/deviceTests/simplecpu/src/com/android/pts/simplecpu/SimpleCpuTest.java
+++ b/suite/pts/deviceTests/simplecpu/src/com/android/pts/simplecpu/SimpleCpuTest.java
@@ -18,6 +18,7 @@
import android.cts.util.TimeoutReq;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.PtsAndroidTestCase;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
@@ -97,7 +98,8 @@
}
getReportLog().printArray("ms", result, false);
Stat.StatResult stat = Stat.getStat(result);
- getReportLog().printSummary("ms", stat.mAverage, stat.mStddev);
+ getReportLog().printSummary("ms", stat.mAverage, PerfResultType.LOWER_BETTER,
+ stat.mStddev);
}
/**
@@ -114,7 +116,8 @@
}
getReportLog().printArray("ms", result, false);
Stat.StatResult stat = Stat.getStat(result);
- getReportLog().printSummary("ms", stat.mAverage, stat.mStddev);
+ getReportLog().printSummary("ms", stat.mAverage, PerfResultType.LOWER_BETTER,
+ stat.mStddev);
}
}
diff --git a/suite/pts/deviceTests/ui/src/com/android/pts/ui/FpsTest.java b/suite/pts/deviceTests/ui/src/com/android/pts/ui/FpsTest.java
index 6931e95..d6d5f06 100644
--- a/suite/pts/deviceTests/ui/src/com/android/pts/ui/FpsTest.java
+++ b/suite/pts/deviceTests/ui/src/com/android/pts/ui/FpsTest.java
@@ -22,6 +22,7 @@
import android.view.Display;
import android.view.WindowManager;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.PtsActivityInstrumentationTestCase2;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
@@ -119,6 +120,8 @@
Log.i(TAG, " fps nominal " + fpsNominal + " fps measured " + fpsMeasured);
getReportLog().printArray("intervals ms", intervals, false);
getReportLog().printArray("jankiness ms", jankiness, false);
- getReportLog().printSummaryFull("Frame interval", " max delay ms", maxDelay, "number of jank", jankNumber);
+ getReportLog().printSummaryFull("Frame interval",
+ "max delay ms", maxDelay, PerfResultType.LOWER_BETTER,
+ "number of jank", jankNumber, PerfResultType.LOWER_BETTER);
}
}
diff --git a/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java b/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
index 8e4bbed..5324669 100644
--- a/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
+++ b/suite/pts/deviceTests/ui/src/com/android/pts/ui/ScrollingTest.java
@@ -18,6 +18,7 @@
import android.cts.util.TimeoutReq;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.PtsActivityInstrumentationTestCase2;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
@@ -67,6 +68,7 @@
});
getReportLog().printArray("ms", results, false);
Stat.StatResult stat = Stat.getStat(results);
- getReportLog().printSummary("Time ms", stat.mAverage, stat.mStddev);
+ getReportLog().printSummary("Time ms", stat.mAverage, PerfResultType.LOWER_BETTER,
+ stat.mStddev);
}
}
diff --git a/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java b/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
index e78289b..2e99bb8 100644
--- a/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
+++ b/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
@@ -19,6 +19,7 @@
import android.cts.util.TimeoutReq;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
import com.android.pts.util.Stat.StatResult;
@@ -67,7 +68,7 @@
});
mReport.printArray("time in ms", result, false);
StatResult stat = Stat.getStat(result);
- mReport.printSummary("time in ms", stat.mAverage, stat.mStddev);
+ mReport.printSummary("time in ms", stat.mAverage, PerfResultType.LOWER_BETTER, stat.mStddev);
}
private void rebootDevice() throws DeviceNotAvailableException {
diff --git a/suite/pts/hostTests/browser/src/com/android/pts/browser/BrowserTest.java b/suite/pts/hostTests/browser/src/com/android/pts/browser/BrowserTest.java
index b5400ab..54f2eb0 100644
--- a/suite/pts/hostTests/browser/src/com/android/pts/browser/BrowserTest.java
+++ b/suite/pts/hostTests/browser/src/com/android/pts/browser/BrowserTest.java
@@ -22,6 +22,7 @@
import com.android.ddmlib.Log;
import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
import com.android.pts.ptsutil.LogcatLineReceiver;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
import com.android.tradefed.build.IBuildInfo;
@@ -98,7 +99,8 @@
}
mReport.printArray("scores", results, true);
Stat.StatResult stat = Stat.getStat(results);
- mReport.printSummary("Score", stat.mAverage, stat.mStddev);
+ // Octane score is higher-better
+ mReport.printSummary("Score", stat.mAverage, PerfResultType.HIGHER_BETTER, stat.mStddev);
}
private double runBenchmarking(String testMethodName, String resultPattern,
diff --git a/suite/pts/hostTests/uihost/control/src/com/android/pts/taskswitching/control/TaskswitchingDeviceTest.java b/suite/pts/hostTests/uihost/control/src/com/android/pts/taskswitching/control/TaskswitchingDeviceTest.java
index fc5bc04..da9d8e2 100644
--- a/suite/pts/hostTests/uihost/control/src/com/android/pts/taskswitching/control/TaskswitchingDeviceTest.java
+++ b/suite/pts/hostTests/uihost/control/src/com/android/pts/taskswitching/control/TaskswitchingDeviceTest.java
@@ -27,6 +27,7 @@
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.PtsAndroidTestCase;
import com.android.pts.util.Stat;
@@ -83,7 +84,8 @@
});
getReportLog().printArray("ms", results, false);
Stat.StatResult stat = Stat.getStat(results);
- getReportLog().printSummary("Time ms", stat.mAverage, stat.mStddev);
+ getReportLog().printSummary("Time ms", stat.mAverage, PerfResultType.LOWER_BETTER,
+ stat.mStddev);
}
private void startActivity(String packageName, String activityName) {
diff --git a/suite/pts/hostTests/uihost/src/com/android/pts/uihost/InstallTimeTest.java b/suite/pts/hostTests/uihost/src/com/android/pts/uihost/InstallTimeTest.java
index 4bfcde1..dedf19c 100644
--- a/suite/pts/hostTests/uihost/src/com/android/pts/uihost/InstallTimeTest.java
+++ b/suite/pts/hostTests/uihost/src/com/android/pts/uihost/InstallTimeTest.java
@@ -24,6 +24,7 @@
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
+import com.android.pts.util.PerfResultType;
import com.android.pts.util.PtsException;
import com.android.pts.util.ReportLog;
import com.android.pts.util.Stat;
@@ -91,7 +92,8 @@
});
mReport.printArray("time in ms", result, false);
StatResult stat = Stat.getStat(result);
- mReport.printSummary("time in ms", stat.mAverage, stat.mStddev);
+ mReport.printSummary("time in ms", stat.mAverage, PerfResultType.LOWER_BETTER,
+ stat.mStddev);
}
}
diff --git a/suite/pts/lib/commonutil/src/com/android/pts/util/PerfResultType.java b/suite/pts/lib/commonutil/src/com/android/pts/util/PerfResultType.java
new file mode 100644
index 0000000..0d0f13d
--- /dev/null
+++ b/suite/pts/lib/commonutil/src/com/android/pts/util/PerfResultType.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.pts.util;
+
+/**
+ * Enum for distinguish performance results.
+ * HIGHER_BETTER: better performance with higher value
+ * LOWER_BETTER: better performance with lower value
+ *
+ */
+public enum PerfResultType {
+ LOWER_BETTER,
+ HIGHER_BETTER;
+}
diff --git a/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java b/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
index 90eed82..721a196 100644
--- a/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
+++ b/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
@@ -39,6 +39,7 @@
private List<String> mMessages = new LinkedList<String> ();
private String mSummary = null;
protected static int mDepth = 3;
+
/**
* print given value to the report
* @param header string to explain the contents. It can be unit for the value.
@@ -76,11 +77,13 @@
* For standard report summary with average and stddev
* @param header
* @param average
+ * @param type type of average value. stddev does not need type.
* @param stddev
*/
- public void printSummary(String header, double average, double stddev) {
- mSummary = header + LOG_ELEM_SEPARATOR + "average " + average + LOG_ELEM_SEPARATOR +
- "stddev " + stddev;
+ public void printSummary(String header, double average, PerfResultType type, double stddev) {
+ mSummary = header + LOG_ELEM_SEPARATOR + "average " + average + " " + type.ordinal() +
+ LOG_ELEM_SEPARATOR + "stddev " + stddev + " " +
+ PerfResultType.LOWER_BETTER.ordinal();
}
/**
@@ -88,12 +91,15 @@
* @param header
* @param key1 String key for val1
* @param val1
+ * @param val1type
* @param key2 String key for val2
* @param val2
+ * @param val2type
*/
- public void printSummaryFull(String header, String key1, double val1, String key2, double val2) {
- mSummary = header + LOG_ELEM_SEPARATOR + key1 + " " + val1 + LOG_ELEM_SEPARATOR +
- key2 + " " + val2;
+ public void printSummaryFull(String header, String key1, double val1, PerfResultType val1type,
+ String key2, double val2, PerfResultType val2type) {
+ mSummary = header + LOG_ELEM_SEPARATOR + key1 + " " + val1 + " " + val1type.ordinal() +
+ LOG_ELEM_SEPARATOR + key2 + " " + val2 + " " + val2type.ordinal();
}
public void throwReportToHost() throws PtsException {
