prod-tests/src/com/android/graphics/tests/FlatlandTest.java

/*
 * Copyright (C) 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.graphics.tests;

import com.android.tradefed.config.Option;
import com.android.tradefed.config.Option.Importance;
import com.android.tradefed.device.CollectingOutputReceiver;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.util.AbiFormatter;
import com.android.tradefed.util.RunUtil;

import junit.framework.Assert;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
 * Test Runner for graphics Flatland Benchmark test.
 * <p>
 * Flatland test is a benchmark for measuring GPU performance in various 2D UI rendering and
 * window composition scenarios.
 * <p>
 * Since it is measuring the hardware performance, the test should be executed in
 * as consistent and static an environment as possible.
 * <ul>
 * <li>The display should be turned off and background service should be stopped before
 * running the benchmark. Running 'adb shell stop' is probably sufficient for this,
 * but if there are device specific background services that consume
 * much CPU cycles, memory bandwidth, or might otherwise interfere with GPU rendering,
 * those should be stopped as well
 * <li>All relevant hardware clocks should be locked at particular frequency when running the test.
 * </ul>
 * <p>
 * If running the benchmark with clocks locked causes thermal throttling, set option "--sleep-time"
 * to 10 to 50 (ms) to insert sleep between each benchmark sample run.
 * <p>
 * Output interpretation:
 * For each test case, the expected time in milliseconds that a single frame of the scenario
 * takes to complete will be printed out. Four types of values could displayed:
 * <ul>
 * <li>fast - frames of the scenarios are completed too fast to be reliably benchmarked. This
 * corresponds to frame time less than 3 ms. The scenario was skipped. "0" will be posted into
 * the dashboard.
 * <li>slow - frame time is too long, normally orver 50 ms. The scenario was skipped. "1000" will
 * be posted into the dashboard.
 * <li>varies - frame time was not stable. rerun the test to get a stable results. If that results
 * show repeatedly, something is wrong with the environment, signal to file a bug.
 * <li>decimal number - frame time for the scenarios are measured.
 * </ul>
 */
public class FlatlandTest implements IDeviceTest, IRemoteTest {

    private static final long SHELL_TIMEOUT = 30*60*1000;
    private static final String COMMAND = "flatland|#ABI32#|";
    private static final String FIRST_LINE = "cmdline:";
    private static final String TITLE = "Scenario";
    private static final long START_TIMER = 2 * 60 * 1000; // 2 minutes
    private static final String RESULT_FAST = "fast";
    private static final String RESULT_SLOW = "slow";
    private static final String RESULT_VARIES = "varies";

    private ITestDevice mTestDevice = null;
    // HashMap to store results for
    public Map<String, String> mResultMap = new HashMap<String, String>();

    @Option(name = "ru-key", description = "Reporting unit key to use when posting results")
    private String mRuKey = "flatland";

    @Option(name = "run-path",
            description = "path for the binary")
    private String mRunPath = "/data/local/tmp/";

    @Option(name = "sleep-time",
            description = "sleep for N ms between samples, set to 10 - 50 ms if the locked CPU"
                    + " frequency causes thermal throttle.")
    private int mSleepTime = 50;

    @Option(name = "schema-map",
            description = "map a test case name to a schema key")
    private Map<String, String> mSchemaMap = new HashMap<String, String>();

    @Option(name = AbiFormatter.FORCE_ABI_STRING,
            description = AbiFormatter.FORCE_ABI_DESCRIPTION,
            importance = Importance.IF_UNSET)
    private String mForceAbi = null;

    @Override
    public void setDevice(ITestDevice testDevice) {
        mTestDevice = testDevice;
    }

    @Override
    public ITestDevice getDevice() {
        return mTestDevice;
    }

    @Override
    public void run(ITestInvocationListener standardListener) throws DeviceNotAvailableException {
        Assert.assertNotNull(mRunPath);
        RunUtil.getDefault().sleep(START_TIMER);

        // execute test
        StringBuilder cmd = new StringBuilder();
        cmd.append(mRunPath);
        cmd.append(COMMAND);
        if (mSleepTime > 0) {
            cmd.append(" -s ");
            cmd.append(mSleepTime);
        }
        standardListener.testRunStarted(mRuKey, 1);
        long start = System.currentTimeMillis();
        CollectingOutputReceiver receiver = new CollectingOutputReceiver();
        mTestDevice.executeShellCommand(AbiFormatter.formatCmdForAbi(cmd.toString(), mForceAbi),
                receiver, SHELL_TIMEOUT, TimeUnit.MILLISECONDS, 2);
        String result = receiver.getOutput();
        if (result == null) {
            CLog.v("no test results returned. Test failed?");
            return;
        }
        // parse results and report metrics
        parseResult(result);
        standardListener.testRunEnded((System.currentTimeMillis() - start), mResultMap);
    }

    /**
     * Parse results returned from running the benchmark
     */
    public void parseResult(String result) {
        String[] lines = result.split(System.getProperty("line.separator"));
        if (lines.length <= 0) {
            return;
        }
        for (int i = 0; i < lines.length; i++) {
            if (!lines[i].contains(FIRST_LINE) && !(lines[i].contains(TITLE))) {
                // skip the first two lines
                String[] items = lines[i].trim().split("\\|");
                if (items.length == 3) {
                    String schemaKey = String.format("%s %s", items[0].trim(), items[1].trim());
                    if (mSchemaMap.get(schemaKey) != null) {
                        // get the mapped schema key if there is any
                        schemaKey = mSchemaMap.get(schemaKey);
                    }
                    String renderTime = items[2].trim();
                    if (renderTime != null) {
                        if (renderTime.equals(RESULT_FAST)) {
                            mResultMap.put(schemaKey, "0");
                        } else if (renderTime.equals(RESULT_SLOW)) {
                            mResultMap.put(schemaKey, "1000");
                        } else if (renderTime.equals(RESULT_VARIES)){
                            mResultMap.put(schemaKey, "-1");
                        } else {
                            mResultMap.put(schemaKey, renderTime);
                        }
                    }
                }
            }
        }
    }
}