Fix startup test setup sequences
This will run camera startup instrumentation test multiple
time to get average of cold startup metrics.
It reboots to clear dirty states in the previous run,
and also waits for device to cool down in between test runs
keeping test condition consistent.
Last, post the average of multiple test runs or -1 if
metric value is out of valid range.
Bug: 27229178
Change-Id: Ia84485d118bf7b50d2bd68bf5dfffff1cb55b6b9
diff --git a/src/com/android/media/tests/CameraStartupTest.java b/src/com/android/media/tests/CameraStartupTest.java
index 8495f4f..ac4e8da 100644
--- a/src/com/android/media/tests/CameraStartupTest.java
+++ b/src/com/android/media/tests/CameraStartupTest.java
@@ -17,13 +17,23 @@
package com.android.media.tests;
import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.targetprep.BuildError;
+import com.android.tradefed.targetprep.ITargetPreparer;
+import com.android.tradefed.targetprep.TargetSetupError;
+import com.android.tradefed.targetprep.TemperatureThrottlingWaiter;
+import com.android.tradefed.util.MultiMap;
+import junit.framework.Assert;
+
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -38,6 +48,23 @@
private static final Pattern STATS_REGEX = Pattern.compile(
"^(?<coldStartup>[0-9.]+)\\|(?<warmStartup>[0-9.]+)\\|(?<values>[0-9 .-]+)");
private static final String PREFIX_COLD_STARTUP = "Cold";
+ // all metrics are expected to be less than 10 mins and greater than 0.
+ private static final int METRICS_MAX_THRESHOLD_MS = 10 * 60 * 1000;
+ private static final int METRICS_MIN_THRESHOLD_MS = 0;
+ private static final String INVALID_VALUE = "-1";
+
+ @Option(name="num-test-runs", description="The number of test runs. A instrumentation "
+ + "test will be repeatedly executed. Then it posts the average of test results.")
+ private int mNumTestRuns = 1;
+
+ @Option(name="delay-between-test-runs", description="Time delay between multiple test runs, "
+ + "in msecs. Used to wait for device to cool down. "
+ + "Note that this will be ignored when TemperatureThrottlingWaiter is configured.")
+ private long mDelayBetweenTestRunsMs = 120 * 1000; // 2 minutes
+
+ private MultiMap<String, String> mMultipleRunMetrics = new MultiMap<String, String>();
+ private Map<String, String> mAverageMultipleRunMetrics = new HashMap<String, String>();
+ private long mTestRunsDurationMs = 0;
public CameraStartupTest() {
setTestPackage("com.google.android.camera");
@@ -52,7 +79,105 @@
*/
@Override
public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
- runInstrumentationTest(listener, new CollectingListener(listener));
+ runMultipleInstrumentationTests(listener, mNumTestRuns);
+ }
+
+ private void runMultipleInstrumentationTests(ITestInvocationListener listener, int numTestRuns)
+ throws DeviceNotAvailableException {
+ Assert.assertTrue(numTestRuns > 0);
+
+ mTestRunsDurationMs = 0;
+ for (int i = 0; i < numTestRuns; ++i) {
+ CLog.v("Running multiple instrumentation tests... [%d/%d]", i + 1, numTestRuns);
+ CollectingListener singleRunListener = new CollectingListener(listener);
+ runInstrumentationTest(listener, singleRunListener);
+ mTestRunsDurationMs += getTestDurationMs();
+
+ if (singleRunListener.hasFailedTests() ||
+ singleRunListener.hasTestRunFatalError()) {
+ exitTestRunsOnError(listener, singleRunListener.getErrorMessage());
+ return;
+ }
+ if (i + 1 < numTestRuns) { // Skipping preparation on the last run
+ postSetupTestRun();
+ }
+ }
+
+ // Post the average of metrics collected in multiple instrumentation test runs.
+ postMultipleRunMetrics(listener);
+ CLog.v("multiple instrumentation tests end");
+ }
+
+ private void exitTestRunsOnError(ITestInvocationListener listener, String errorMessage) {
+ CLog.e("The instrumentation result not found. Test runs may have failed due to exceptions."
+ + " Test results will not be posted. errorMsg: %s", errorMessage);
+ listener.testRunFailed(errorMessage);
+ listener.testRunEnded(mTestRunsDurationMs, Collections.<String, String>emptyMap());
+ }
+
+ private void postMultipleRunMetrics(ITestInvocationListener listener) {
+ listener.testRunEnded(mTestRunsDurationMs, getAverageMultipleRunMetrics());
+ }
+
+ private void postSetupTestRun() throws DeviceNotAvailableException {
+ // Reboot for a cold start up of Camera application
+ CLog.d("Cold start: Rebooting...");
+ getDevice().reboot();
+
+ // Wait for device to cool down to target temperature
+ // Use TemperatureThrottlingWaiter if configured, otherwise just wait for
+ // a specific amount of time.
+ CLog.d("Cold start: Waiting for device to cool down...");
+ boolean usedTemperatureThrottlingWaiter = false;
+ for (ITargetPreparer preparer : mConfiguration.getTargetPreparers()) {
+ if (preparer instanceof TemperatureThrottlingWaiter) {
+ usedTemperatureThrottlingWaiter = true;
+ try {
+ preparer.setUp(getDevice(), null);
+ } catch (TargetSetupError e) {
+ CLog.w("No-op even when temperature is still high after wait timeout. "
+ + "error: %s", e.getMessage());
+ } catch (BuildError e) {
+ // This should not happen.
+ }
+ }
+ }
+ if (!usedTemperatureThrottlingWaiter) {
+ getRunUtil().sleep(mDelayBetweenTestRunsMs);
+ }
+ CLog.d("Device gets prepared for the next test run.");
+ }
+
+ // Call this function once at the end to get the average.
+ private Map<String, String> getAverageMultipleRunMetrics() {
+ Assert.assertTrue(mMultipleRunMetrics.size() > 0);
+
+ Set<String> keys = mMultipleRunMetrics.keySet();
+ mAverageMultipleRunMetrics.clear();
+ for (String key : keys) {
+ int sum = 0;
+ int size = 0;
+ boolean isInvalid = false;
+ for (String valueString : mMultipleRunMetrics.get(key)) {
+ int value = Integer.parseInt(valueString);
+ // If value is out of valid range, skip posting the result associated with the key
+ if (value > METRICS_MAX_THRESHOLD_MS || value < METRICS_MIN_THRESHOLD_MS) {
+ isInvalid = true;
+ break;
+ }
+ sum += value;
+ ++size;
+ }
+
+ String valueString = INVALID_VALUE;
+ if (isInvalid) {
+ CLog.w("Value is out of valid range. Key: %s ", key);
+ } else {
+ valueString = String.format("%d", (sum / size));
+ }
+ mAverageMultipleRunMetrics.put(key, valueString);
+ }
+ return mAverageMultipleRunMetrics;
}
/**
@@ -70,6 +195,14 @@
getAggregatedMetrics().putAll(parseResults(testMetrics));
}
+ @Override
+ public void handleTestRunEnded(ITestInvocationListener listener, long elapsedTime,
+ Map<String, String> runMetrics) {
+ // Do not post aggregated metrics from a single run to a dashboard. Instead, it needs
+ // to collect all metrics from multiple test runs.
+ mMultipleRunMetrics.putAll(getAggregatedMetrics());
+ }
+
public Map<String, String> parseResults(Map<String, String> testMetrics) {
// Parse activity time stats from the instrumentation result.
// Format : <metric_key>=<cold_startup>|<average_of_warm_startups>|<all_startups>
@@ -87,13 +220,15 @@
Map<String, String> parsed = new HashMap<String, String>();
for (Map.Entry<String, String> metric : testMetrics.entrySet()) {
Matcher matcher = STATS_REGEX.matcher(metric.getValue());
+ String keyName = metric.getKey();
+ String coldStartupValue = INVALID_VALUE;
+ String warmStartupValue = INVALID_VALUE;
if (matcher.matches()) {
- String keyName = metric.getKey();
- parsed.put(PREFIX_COLD_STARTUP + keyName, matcher.group("coldStartup"));
- parsed.put(keyName, matcher.group("warmStartup"));
- } else {
- CLog.w(String.format("Stats not in correct format: %s", metric.getValue()));
+ coldStartupValue = matcher.group("coldStartup");
+ warmStartupValue = matcher.group("warmStartup");
}
+ parsed.put(PREFIX_COLD_STARTUP + keyName, coldStartupValue);
+ parsed.put(keyName, warmStartupValue);
}
return parsed;
}
diff --git a/src/com/android/media/tests/CameraTestBase.java b/src/com/android/media/tests/CameraTestBase.java
index a1c43d0..0c4a734 100644
--- a/src/com/android/media/tests/CameraTestBase.java
+++ b/src/com/android/media/tests/CameraTestBase.java
@@ -18,6 +18,8 @@
import com.android.ddmlib.CollectingOutputReceiver;
import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.IConfigurationReceiver;
import com.android.tradefed.config.Option;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
@@ -32,6 +34,8 @@
import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.testtype.InstrumentationTest;
import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunUtil;
import junit.framework.Assert;
@@ -57,16 +61,16 @@
* Camera2StressTest, CameraStartupTest, Camera2LatencyTest and CameraPerformanceTest use this base
* class for Camera ivvavik and later.
*/
-public class CameraTestBase implements IDeviceTest, IRemoteTest {
+public class CameraTestBase implements IDeviceTest, IRemoteTest, IConfigurationReceiver {
private static final String LOG_TAG = CameraTestBase.class.getSimpleName();
private static final long SHELL_TIMEOUT_MS = 60 * 1000; // 1 min
private static final int SHELL_MAX_ATTEMPTS = 3;
- private static final String PROCESS_CAMERA_DAEMON = "mm-qcamera-daemon";
- private static final String PROCESS_MEDIASERVER = "mediaserver";
- private static final String PROCESS_CAMERA_APP = "com.google.android.GoogleCamera";
- private static final String DUMP_ION_HEAPS_COMMAND = "cat /d/ion/heaps/system";
- private static final String ARGUMENT_TEST_ITERATIONS = "iterations";
+ protected static final String PROCESS_CAMERA_DAEMON = "mm-qcamera-daemon";
+ protected static final String PROCESS_MEDIASERVER = "mediaserver";
+ protected static final String PROCESS_CAMERA_APP = "com.google.android.GoogleCamera";
+ protected static final String DUMP_ION_HEAPS_COMMAND = "cat /d/ion/heaps/system";
+ protected static final String ARGUMENT_TEST_ITERATIONS = "iterations";
@Option(name = "test-package", description = "Test package to run.")
private String mTestPackage = "com.google.android.camera";
@@ -137,6 +141,8 @@
private MeminfoTimer mMeminfoTimer = null;
private ThreadTrackerTimer mThreadTrackerTimer = null;
+ protected IConfiguration mConfiguration;
+
/**
* {@inheritDoc}
*/
@@ -302,6 +308,12 @@
}
@Override
+ public void testRunFailed(String errorMessage) {
+ super.testRunFailed(errorMessage);
+ mFatalErrors.put(getRuKey(), errorMessage);
+ }
+
+ @Override
public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) {
super.testRunEnded(elapsedTime, runMetrics);
handleTestRunEnded(mListener, elapsedTime, runMetrics);
@@ -736,6 +748,23 @@
}
/**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setConfiguration(IConfiguration configuration) {
+ mConfiguration = configuration;
+ }
+
+ /**
+ * Get the {@link IRunUtil} instance to use.
+ * <p/>
+ * Exposed so unit tests can mock.
+ */
+ IRunUtil getRunUtil() {
+ return RunUtil.getDefault();
+ }
+
+ /**
* Get the duration of Camera test instrumentation in milliseconds.
*
* @return the duration of Camera instrumentation test until it is called.
