Add Flatland benchmark runner
Change-Id: Ib5baed34db4d11e0e2deae2406bb26567e6ea340
diff --git a/prod-tests/src/com/android/graphics/tests/FlatlandTest.java b/prod-tests/src/com/android/graphics/tests/FlatlandTest.java
new file mode 100644
index 0000000..1d47487
--- /dev/null
+++ b/prod-tests/src/com/android/graphics/tests/FlatlandTest.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.graphics.tests;
+
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.RunUtil;
+
+import junit.framework.Assert;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Test Runner for graphics Flatland Benchmark test.
+ * <p>
+ * Flatland test is a benchmark for measuring GPU performance in various 2D UI rendering and
+ * window composition scenarios.
+ * <p>
+ * Since it is measuring the hardware performance, the test should be executed in
+ * as consistent and static an environment as possible.
+ * <ul>
+ * <li>The display should be turned off and background service should be stopped before
+ * running the benchmark. Running 'adb shell stop' is probably sufficient for this,
+ * but if there are device specific background services that consume
+ * much CPU cycles, memory bandwidth, or might otherwise interfere with GPU rendering,
+ * those should be stopped as well
+ * <li>All relevant hardware clocks should be locked at particular frequency when running the test.
+ * </ul>
+ * <p>
+ * If running the benchmark with clocks locked causes thermal throttling, set option "--sleep-time"
+ * to 10 to 50 (ms) to insert sleep between each benchmark sample run.
+ * <p>
+ * Output interpretation:
+ * For each test case, the expected time in milliseconds that a single frame of the scenario
+ * takes to complete will be printed out. Four types of values could displayed:
+ * <ul>
+ * <li>fast - frames of the scenarios are completed too fast to be reliably benchmarked. This
+ * corresponds to frame time less than 3 ms. The scenario was skipped. "0" will be posted into
+ * the dashboard.
+ * <li>slow - frame time is too long, normally orver 50 ms. The scenario was skipped. "1000" will
+ * be posted into the dashboard.
+ * <li>varies - frame time was not stable. rerun the test to get a stable results. If that results
+ * show repeatedly, something is wrong with the environment, signal to file a bug.
+ * <li>decimal number - frame time for the scenarios are measured.
+ * </ul>
+ */
+public class FlatlandTest implements IDeviceTest, IRemoteTest {
+ private static final String COMMAND = "flatland";
+ private static final String FIRST_LINE = "cmdline:";
+ private static final String TITLE = "Scenario";
+ private static final long START_TIMER = 2 * 60 * 1000; // 2 minutes
+ private static final String RESULT_FAST = "fast";
+ private static final String RESULT_SLOW = "slow";
+ private static final String RESULT_VARIES = "varies";
+
+ private ITestDevice mTestDevice = null;
+ // HashMap to store results for
+ public Map<String, String> mResultMap = new HashMap<String, String>();
+
+ @Option(name = "ru-key", description = "Reporting unit key to use when posting results")
+ private String mRuKey = "flatland";
+
+ @Option(name = "run-path",
+ description = "path for the binary")
+ private String mRunPath = "/data/local/tmp/";
+
+ @Option(name = "sleep-time",
+ description = "sleep for N ms between samples, set to 10 - 50 ms if the locked CPU"
+ + " frequency causes thermal throttle.")
+ private int mSleepTime = 50;
+
+ @Option(name = "schema-map",
+ description = "map a test case name to a schema key")
+ private Map<String, String> mSchemaMap = new HashMap<String, String>();
+
+ @Override
+ public void setDevice(ITestDevice testDevice) {
+ mTestDevice = testDevice;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+
+ @Override
+ public void run(ITestInvocationListener standardListener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mRunPath);
+ RunUtil.getDefault().sleep(START_TIMER);
+
+ // execute test
+ StringBuilder cmd = new StringBuilder();
+ cmd.append(mRunPath);
+ cmd.append(COMMAND);
+ if (mSleepTime > 0) {
+ cmd.append(" -s ");
+ cmd.append(mSleepTime);
+ }
+ standardListener.testRunStarted(mRuKey, 1);
+ long start = System.currentTimeMillis();
+ String result = mTestDevice.executeShellCommand(cmd.toString());
+ if (result == null) {
+ CLog.v("no test results returned. Test failed?");
+ return;
+ }
+ // parse results and report metrics
+ parseResult(result);
+ standardListener.testRunEnded((System.currentTimeMillis() - start), mResultMap);
+ }
+
+ /**
+ * Parse results returned from running the benchmark
+ */
+ public void parseResult(String result) {
+ String[] lines = result.split(System.getProperty("line.separator"));
+ if (lines.length <= 0) {
+ return;
+ }
+ for (int i = 0; i < lines.length; i++) {
+ if (!lines[i].contains(FIRST_LINE) && !(lines[i].contains(TITLE))) {
+ // skip the first two lines
+ String[] items = lines[i].trim().split("\\|");
+ if (items.length == 3) {
+ String schemaKey = String.format("%s %s", items[0].trim(), items[1].trim());
+ if (mSchemaMap.get(schemaKey) != null) {
+ // get the mapped schema key if there is any
+ schemaKey = mSchemaMap.get(schemaKey);
+ }
+ String renderTime = items[2].trim();
+ if (renderTime != null) {
+ if (renderTime.equals(RESULT_FAST)) {
+ mResultMap.put(schemaKey, "0");
+ } else if (renderTime.equals(RESULT_SLOW)) {
+ mResultMap.put(schemaKey, "1000");
+ } else if (renderTime.equals(RESULT_VARIES)){
+ mResultMap.put(schemaKey, "-1");
+ } else {
+ mResultMap.put(schemaKey, renderTime);
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/prod-tests/src/com/android/graphics/tests/FlatlandTestFuncTest.java b/prod-tests/src/com/android/graphics/tests/FlatlandTestFuncTest.java
new file mode 100644
index 0000000..da837a2
--- /dev/null
+++ b/prod-tests/src/com/android/graphics/tests/FlatlandTestFuncTest.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.graphics.tests;
+
+import com.android.tradefed.log.LogUtil.CLog;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import java.util.Map;
+
+public class FlatlandTestFuncTest extends TestCase {
+
+ private final String output = " cmdline: /data/local/tmp/flatland\n" +
+ " Scenario | Resolution | Time (ms)\n" +
+ " 16:10 Single Static Window | 1280 x 800 | fast\n" +
+ " 16:10 Single Static Window | 1920 x 1200 | 3.136\n" +
+ " 16:10 Single Static Window | 2560 x 1600 | 5.524\n" +
+ " 16:10 Single Static Window | 3840 x 2400 | 11.841\n" +
+ " 4:3 Single Static Window | 2048 x 1536 | 4.292\n" +
+ " 16:10 App -> Home Transition | 1280 x 800 | varies\n" +
+ " 16:10 App -> Home Transition | 1920 x 1200 | 5.724\n" +
+ " 16:10 App -> Home Transition | 2560 x 1600 | 10.033\n" +
+ " 16:10 App -> Home Transition | 3840 x 2400 | 22.034\n" +
+ " 4:3 App -> Home Transition | 2048 x 1536 | 8.003\n" +
+ " 16:10 SurfaceView -> Home Transition | 1280 x 800 | slow\n" +
+ " 16:10 SurfaceView -> Home Transition | 1920 x 1200 | 7.023\n" +
+ " 16:10 SurfaceView -> Home Transition | 2560 x 1600 | 12.337\n" +
+ " 16:10 SurfaceView -> Home Transition | 3840 x 2400 | 27.283\n" +
+ " 4:3 SurfaceView -> Home Transition | 2048 x 1536 | 9.918\n";
+
+ public void testPraseResults() throws Exception {
+ FlatlandTest ft = new FlatlandTest();
+ ft.parseResult(output);
+ // "0" represents a "fast" result
+ String t = ft.mResultMap.get("16:10 Single Static Window 1280 x 800");
+ Assert.assertTrue(t.equals("0"));
+
+ t = ft.mResultMap.get("16:10 Single Static Window 1920 x 1200");
+ Assert.assertTrue(t.equals("3.136"));
+
+ t = ft.mResultMap.get("16:10 App -> Home Transition 1280 x 800");
+ Assert.assertTrue(t.equals("-1"));
+
+ t = ft.mResultMap.get("16:10 SurfaceView -> Home Transition 1280 x 800");
+ Assert.assertTrue(t.equals("1000"));
+
+ for(Map.Entry<String, String> entry: ft.mResultMap.entrySet()) {
+ CLog.e("key:%s, value:%s", entry.getKey(), entry.getValue());
+ }
+ }
+}
