Make perf metrics collection & output via instrumentation simpler
- Added includeDetailedStats metadata to TimedTest for collecting additional
metrics like memory usage and binder transactions
- Added PerformanceCollectorTestCase interface for hard-typing test class as
performance test while still able to inherit from instrumenation test classes
- Reverted previous changes to PerformanceTestBase, will deprecate
PerformanceTestBase/Case in a future CL
- Removed 'performance.' prefix from keys written to instrumentation output
diff --git a/test-runner/android/test/AndroidTestRunner.java b/test-runner/android/test/AndroidTestRunner.java
index 0f1599a..fc9832c 100644
--- a/test-runner/android/test/AndroidTestRunner.java
+++ b/test-runner/android/test/AndroidTestRunner.java
@@ -133,7 +133,7 @@
}
return new TestResult();
}
-
+
void setSkipExecution(boolean skip) {
mSkipExecution = skip;
}
@@ -165,7 +165,7 @@
for (TestCase testCase : mTestCases) {
setContextIfAndroidTestCase(testCase, mContext, testContext);
setInstrumentationIfInstrumentationTestCase(testCase, mInstrumentation);
- setPerformanceWriterIfPerformanceTestCase(testCase, mPerfWriter);
+ setPerformanceWriterIfPerformanceCollectorTestCase(testCase, mPerfWriter);
testCase.run(mTestResult);
}
}
@@ -188,10 +188,10 @@
}
}
- private void setPerformanceWriterIfPerformanceTestCase(
+ private void setPerformanceWriterIfPerformanceCollectorTestCase(
Test test, PerformanceResultsWriter writer) {
- if (PerformanceTestBase.class.isAssignableFrom(test.getClass())) {
- ((PerformanceTestBase) test).setPerformanceResultsWriter(writer);
+ if (PerformanceCollectorTestCase.class.isAssignableFrom(test.getClass())) {
+ ((PerformanceCollectorTestCase) test).setPerformanceResultsWriter(writer);
}
}
diff --git a/test-runner/android/test/InstrumentationTestRunner.java b/test-runner/android/test/InstrumentationTestRunner.java
index 773d7a9..3e9cd9f 100644
--- a/test-runner/android/test/InstrumentationTestRunner.java
+++ b/test-runner/android/test/InstrumentationTestRunner.java
@@ -27,8 +27,6 @@
import android.os.Looper;
import android.os.Parcelable;
import android.os.PerformanceCollector;
-import android.os.Process;
-import android.os.SystemClock;
import android.os.PerformanceCollector.PerformanceResultsWriter;
import android.test.suitebuilder.TestMethod;
import android.test.suitebuilder.TestPredicates;
@@ -226,23 +224,6 @@
* identifies the path to the generated code coverage file.
*/
private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath";
- /**
- * If included at the start of reporting keys, this prefix marks the key as a performance
- * metric.
- */
- private static final String REPORT_KEY_PREFIX = "performance.";
- /**
- * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
- * reports the cpu time in milliseconds of the current test.
- */
- private static final String REPORT_KEY_PERF_CPU_TIME =
- REPORT_KEY_PREFIX + PerformanceCollector.METRIC_KEY_CPU_TIME;
- /**
- * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
- * reports the run time in milliseconds of the current test.
- */
- private static final String REPORT_KEY_PERF_EXECUTION_TIME =
- REPORT_KEY_PREFIX + PerformanceCollector.METRIC_KEY_EXECUTION_TIME;
/**
* The test is starting.
@@ -630,9 +611,9 @@
int mTestNum = 0;
int mTestResultCode = 0;
String mTestClass = null;
+ PerformanceCollector mPerfCollector = new PerformanceCollector();
boolean mIsTimedTest = false;
- long mCpuTime = 0;
- long mExecTime = 0;
+ boolean mIncludeDetailedStats = false;
public WatcherResultPrinter(int numTests) {
mResultTemplate = new Bundle();
@@ -675,20 +656,28 @@
mTestResultCode = 0;
mIsTimedTest = false;
+ mIncludeDetailedStats = false;
try {
- // Look for TimedTest annotation on both test class and test
- // method
- mIsTimedTest = test.getClass().isAnnotationPresent(TimedTest.class) ||
- test.getClass().getMethod(testName).isAnnotationPresent(TimedTest.class);
+ // Look for TimedTest annotation on both test class and test method
+ if (test.getClass().getMethod(testName).isAnnotationPresent(TimedTest.class)) {
+ mIsTimedTest = true;
+ mIncludeDetailedStats = test.getClass().getMethod(testName).getAnnotation(
+ TimedTest.class).includeDetailedStats();
+ } else if (test.getClass().isAnnotationPresent(TimedTest.class)) {
+ mIsTimedTest = true;
+ mIncludeDetailedStats = test.getClass().getAnnotation(
+ TimedTest.class).includeDetailedStats();
+ }
} catch (SecurityException e) {
throw new IllegalStateException(e);
} catch (NoSuchMethodException e) {
throw new IllegalStateException(e);
}
- if (mIsTimedTest) {
- mExecTime = SystemClock.uptimeMillis();
- mCpuTime = Process.getElapsedCpuTime();
+ if (mIsTimedTest && mIncludeDetailedStats) {
+ mPerfCollector.beginSnapshot("");
+ } else if (mIsTimedTest) {
+ mPerfCollector.startTiming("");
}
}
@@ -720,11 +709,10 @@
* @see junit.framework.TestListener#endTest(Test)
*/
public void endTest(Test test) {
- if (mIsTimedTest) {
- mCpuTime = Process.getElapsedCpuTime() - mCpuTime;
- mExecTime = SystemClock.uptimeMillis() - mExecTime;
- mTestResult.putLong(REPORT_KEY_PERF_CPU_TIME, mCpuTime);
- mTestResult.putLong(REPORT_KEY_PERF_EXECUTION_TIME, mExecTime);
+ if (mIsTimedTest && mIncludeDetailedStats) {
+ mTestResult.putAll(mPerfCollector.endSnapshot());
+ } else if (mIsTimedTest) {
+ writeStopTiming(mPerfCollector.stopTiming(""));
}
if (mTestResultCode == 0) {
@@ -760,7 +748,7 @@
for (Parcelable p :
results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) {
Bundle iteration = (Bundle)p;
- String index = "performance.iteration" + i + ".";
+ String index = "iteration" + i + ".";
mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL,
iteration.getString(PerformanceCollector.METRIC_KEY_LABEL));
mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME,
@@ -772,15 +760,15 @@
}
public void writeMeasurement(String label, long value) {
- mTestResult.putLong(REPORT_KEY_PREFIX + label, value);
+ mTestResult.putLong(label, value);
}
public void writeMeasurement(String label, float value) {
- mTestResult.putFloat(REPORT_KEY_PREFIX + label, value);
+ mTestResult.putFloat(label, value);
}
public void writeMeasurement(String label, String value) {
- mTestResult.putString(REPORT_KEY_PREFIX + label, value);
+ mTestResult.putString(label, value);
}
// TODO report the end of the cycle
diff --git a/test-runner/android/test/PerformanceCollectorTestCase.java b/test-runner/android/test/PerformanceCollectorTestCase.java
new file mode 100644
index 0000000..4309ff7
--- /dev/null
+++ b/test-runner/android/test/PerformanceCollectorTestCase.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.test;
+
+import android.os.PerformanceCollector;
+import android.os.PerformanceCollector.PerformanceResultsWriter;
+
+/**
+ * A simple interface for passing in a PerformanceResultsWriter instance to be used with
+ * PerformanceCollector.
+ * <p/>
+ * A one line implementation of {@link #setPerformanceResultsWriter(PerformanceResultsWriter)}
+ * is sufficient in most cases:
+ * <p/>
+ * <code>mPerfCollector.setPerformanceResultsWriter(writer);</code>
+ *
+ * {@hide} Not needed for SDK.
+ */
+public interface PerformanceCollectorTestCase {
+ public PerformanceCollector mPerfCollector = new PerformanceCollector();
+
+ public void setPerformanceResultsWriter(PerformanceResultsWriter writer);
+}
diff --git a/test-runner/android/test/PerformanceTestBase.java b/test-runner/android/test/PerformanceTestBase.java
index 572a9b8..4a0a589 100644
--- a/test-runner/android/test/PerformanceTestBase.java
+++ b/test-runner/android/test/PerformanceTestBase.java
@@ -16,95 +16,12 @@
package android.test;
-import android.os.Bundle;
-import android.os.PerformanceCollector;
-import android.os.PerformanceCollector.PerformanceResultsWriter;
-
-import java.lang.reflect.Method;
+import junit.framework.TestCase;
/**
- * Provides hooks and wrappers to automatically and manually collect and report
- * performance data in tests.
- *
- * {@hide} Pending approval for public API.
+ * {@hide} Not needed for SDK.
*/
-public class PerformanceTestBase extends InstrumentationTestCase implements PerformanceTestCase {
-
- private static PerformanceCollector sPerfCollector = new PerformanceCollector();
- private static int sNumTestMethods = 0;
- private static int sNumTestMethodsLeft = 0;
-
- // Count number of tests, used to emulate beforeClass and afterClass from JUnit4
- public PerformanceTestBase() {
- if (sNumTestMethods == 0) {
- Method methods[] = getClass().getMethods();
- for (Method m : methods) {
- if (m.getName().startsWith("test")) {
- sNumTestMethods ++;
- sNumTestMethodsLeft ++;
- }
- }
- }
- }
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- // @beforeClass
- // Will skew timing measured by TestRunner, but not by PerformanceCollector
- if (sNumTestMethodsLeft == sNumTestMethods) {
- sPerfCollector.beginSnapshot(this.getClass().getName());
- }
- }
-
- @Override
- protected void tearDown() throws Exception {
- // @afterClass
- // Will skew timing measured by TestRunner, but not by PerformanceCollector
- if (--sNumTestMethodsLeft == 0) {
- sPerfCollector.endSnapshot();
- }
- super.tearDown();
- }
-
- public void setPerformanceResultsWriter(PerformanceResultsWriter writer) {
- sPerfCollector.setPerformanceResultsWriter(writer);
- }
-
- /**
- * @see PerformanceCollector#beginSnapshot(String)
- */
- protected void beginSnapshot(String label) {
- sPerfCollector.beginSnapshot(label);
- }
-
- /**
- * @see PerformanceCollector#endSnapshot()
- */
- protected Bundle endSnapshot() {
- return sPerfCollector.endSnapshot();
- }
-
- /**
- * @see PerformanceCollector#startTiming(String)
- */
- protected void startTiming(String label) {
- sPerfCollector.startTiming(label);
- }
-
- /**
- * @see PerformanceCollector#addIteration(String)
- */
- protected Bundle addIteration(String label) {
- return sPerfCollector.addIteration(label);
- }
-
- /**
- * @see PerformanceCollector#stopTiming(String)
- */
- protected Bundle stopTiming(String label) {
- return sPerfCollector.stopTiming(label);
- }
+public class PerformanceTestBase extends TestCase implements PerformanceTestCase {
public int startPerformance(PerformanceTestCase.Intermediates intermediates) {
return 0;
diff --git a/core/java/android/test/TimedTest.java b/test-runner/android/test/TimedTest.java
similarity index 62%
rename from core/java/android/test/TimedTest.java
rename to test-runner/android/test/TimedTest.java
index 3a60a25..95cc9bf 100644
--- a/core/java/android/test/TimedTest.java
+++ b/test-runner/android/test/TimedTest.java
@@ -20,13 +20,17 @@
import java.lang.annotation.RetentionPolicy;
/**
- * This annotation can be used on an {@link junit.framework.TestCase}'s test
- * methods. When the annotation is present, the test method is timed and the
- * results written through instrumentation output. It can also be used on the
- * class itself, which is equivalent to tagging all test methods with this
- * annotation.
+ * This annotation can be used on an {@link junit.framework.TestCase}'s test methods. When the
+ * annotation is present, the test method is timed and the results written through instrumentation
+ * output. It can also be used on the class itself, which is equivalent to tagging all test methods
+ * with this annotation.
+ * <p/>
+ * Setting {@link #includeDetailedStats()} to true causes additional metrics such as memory usage
+ * and binder transactions to be gathered and written through instrumentation output.
*
* {@hide} Pending approval for public API.
*/
@Retention(RetentionPolicy.RUNTIME)
-public @interface TimedTest { }
\ No newline at end of file
+public @interface TimedTest {
+ boolean includeDetailedStats() default false;
+}
