src/com/android/media/tests/CameraLatencyTest.java

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.media.tests;

import com.android.ddmlib.IDevice;
import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.CollectingTestListener;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.InputStreamSource;
import com.android.tradefed.result.LogDataType;
import com.android.tradefed.result.SnapshotInputStreamSource;
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.util.RegexTrie;
import com.android.tradefed.util.StreamUtil;

import junit.framework.Assert;
import junit.framework.TestCase;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;

/**
 * Runs the Camera latency testcases.
 * FIXME: more details
 * <p/>
 * Note that this test will not run properly unless /sdcard is mounted and writable.
 */
public class CameraLatencyTest implements IDeviceTest, IRemoteTest {
    ITestDevice mTestDevice = null;

    // Constants for running the tests
    private static final String TEST_PACKAGE_NAME = "com.google.android.gallery3d.tests";

    private final String mOutputPath = "mediaStressOut.txt";

    //Max timeout for the test - 30 mins
    private static final int MAX_TEST_TIMEOUT = 30 * 60 * 1000;

    /**
     * Stores the test cases that we should consider running.
     * <p/>
     * This currently consists of "startup" and "latency"
     */
    private List<TestInfo> mTestCases = new ArrayList<TestInfo>();

    /**
     * A struct that contains useful info about the tests to run
     */
    static class TestInfo {
        public String mTestName = null;
        public String mClassName = null;
        public String mTestMetricsName = null;
        public RegexTrie<String> mPatternMap = new RegexTrie<String>();

        @Override
        public String toString() {
            return String.format("TestInfo: name(%s) class(%s) metric(%s) patterns(%s)", mTestName,
                    mClassName, mTestMetricsName, mPatternMap);
        }
    }

    /**
     * Set up the configurations for the test cases we want to run
     */
    public CameraLatencyTest() {
        // Startup tests
        TestInfo t = new TestInfo();
        t.mTestName = "startup";
        t.mClassName = "com.android.gallery3d.stress.CameraStartUp";
        t.mTestMetricsName = "CameraVideoRecorderStartup";
        RegexTrie<String> map = t.mPatternMap;
        map = t.mPatternMap;
        map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)");
        map.put("CameraStartup", "^Camera average startup time: (\\d+) ms");
        map.put("FirstVideoStartup", "^First Video Startup: (\\d+)");
        map.put("VideoStartup", "^Video average startup time: (\\d+) ms");
        mTestCases.add(t);

        // Latency tests
        t = new TestInfo();
        t.mTestName = "latency";
        t.mClassName = "com.android.gallery3d.stress.CameraLatency";
        t.mTestMetricsName = "CameraLatency";
        map = t.mPatternMap;
        map.put("AutoFocus", "^Avg AutoFocus = (\\d+)");
        map.put("ShutterLag", "^Avg mShutterLag = (\\d+)");
        map.put("Preview", "^Avg mShutterToPictureDisplayedTime = (\\d+)");
        map.put("RawPictureGeneration", "^Avg mPictureDisplayedToJpegCallbackTime = (\\d+)");
        map.put("GenTimeDiffOverJPEGAndRaw", "^Avg mJpegCallbackFinishTime = (\\d+)");
        mTestCases.add(t);

    }

    @Override
    public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
        Assert.assertNotNull(mTestDevice);

        for (TestInfo test : mTestCases) {
            cleanTmpFiles();
            executeTest(test, listener);
            logOutputFile(test, listener);
        }

        cleanTmpFiles();
    }

    private void executeTest(TestInfo test, ITestInvocationListener listener)
            throws DeviceNotAvailableException {
        IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME,
                mTestDevice.getIDevice());
        CollectingTestListener auxListener = new CollectingTestListener();

        runner.setClassName(test.mClassName);
        runner.setMaxtimeToOutputResponse(MAX_TEST_TIMEOUT);
        mTestDevice.runInstrumentationTests(runner, listener, auxListener);

        // Grab a bugreport if warranted
        if (auxListener.hasFailedTests()) {
            CLog.i("Grabbing bugreport after test '%s' finished with %d failures and %d errors.",
                    test.mTestName, auxListener.getNumFailedTests(),
                    auxListener.getNumErrorTests());
            InputStreamSource bugreport = mTestDevice.getBugreport();
            listener.testLog(String.format("bugreport-%s.txt", test.mTestName), LogDataType.TEXT,
                    bugreport);
            bugreport.cancel();
        }
    }

    /**
     * Clean up temp files from test runs
     * <p />
     * Note that all photos on the test device will be removed
     */
    private void cleanTmpFiles() throws DeviceNotAvailableException {
        String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
        mTestDevice.executeShellCommand(String.format("rm -r %s/DCIM", extStore));
        mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath));
    }

    /**
     * Pull the output file from the device, add it to the logs, and also parse out the relevant
     * test metrics and report them.
     */
    private void logOutputFile(TestInfo test, ITestInvocationListener listener)
            throws DeviceNotAvailableException {
        File outputFile = null;
        InputStreamSource outputSource = null;
        try {
            outputFile = mTestDevice.pullFileFromExternal(mOutputPath);

            if (outputFile == null) {
                return;
            }

            // Upload a verbatim copy of the output file
            CLog.d("Sending %d byte file %s into the logosphere!", outputFile.length(), outputFile);
            outputSource = new SnapshotInputStreamSource(new FileInputStream(outputFile));
            listener.testLog(String.format("output-%s.txt", test.mTestName), LogDataType.TEXT,
                    outputSource);

            // Parse the output file to upload aggregated metrics
            parseOutputFile(test, new FileInputStream(outputFile), listener);
        } catch (IOException e) {
            CLog.e("IOException while reading or parsing output file: %s", e);
        } finally {
            if (outputFile != null) {
                outputFile.delete();
            }
            if (outputSource != null) {
                outputSource.cancel();
            }
        }
    }

    /**
     * Parse the relevant metrics from the Instrumentation test output file
     */
    private void parseOutputFile(TestInfo test, InputStream dataStream,
            ITestInvocationListener listener) {
        Map<String, String> runMetrics = new HashMap<String, String>();

        // try to parse it
        String contents;
        try {
            contents = StreamUtil.getStringFromStream(dataStream);
        } catch (IOException e) {
            CLog.e("Got IOException during %s test processing: %s", test.mTestName, e);
            return;
        }

        List<String> lines = Arrays.asList(contents.split("\n"));
        ListIterator<String> lineIter = lines.listIterator();
        String line;
        while (lineIter.hasNext()) {
            line = lineIter.next();
            List<List<String>> capture = new ArrayList<List<String>>(1);
            String key = test.mPatternMap.retrieve(capture, line);
            if (key != null) {
                CLog.d("Got %s key '%s' and captures '%s'", test.mTestName, key,
                        capture.toString());
            } else if (line.isEmpty()) {
                // ignore
                continue;
            } else {
                CLog.d("Got unmatched line: %s", line);
                continue;
            }

            runMetrics.put(key, capture.get(0).get(0));
        }

        reportMetrics(listener, test, runMetrics);
    }

    /**
     * Report run metrics by creating an empty test run to stick them in
     * <p />
     * Exposed for unit testing
     */
    void reportMetrics(ITestInvocationListener listener, TestInfo test,
            Map<String, String> metrics) {
        // Create an empty testRun to report the parsed runMetrics
        CLog.d("About to report metrics for %s: %s", test.mTestMetricsName, metrics);
        listener.testRunStarted(test.mTestMetricsName, 0);
        listener.testRunEnded(0, metrics);
    }

    @Override
    public void setDevice(ITestDevice device) {
        mTestDevice = device;
    }

    @Override
    public ITestDevice getDevice() {
        return mTestDevice;
    }

    /**
     * A meta-test to ensure that bits of the CameraLatencyTest are working properly
     */
    public static class MetaTest extends TestCase {
        private CameraLatencyTest mTestInstance = null;

        private TestInfo mTestInfo = null;

        private TestInfo mReportedTestInfo = null;
        private Map<String, String> mReportedMetrics = null;

        private static String join(String... pieces) {
            StringBuilder sb = new StringBuilder();
            for (String piece : pieces) {
                sb.append(piece);
                sb.append("\n");
            }
            return sb.toString();
        }

        @Override
        public void setUp() throws Exception {
            mTestInstance = new CameraLatencyTest() {
                @Override
                void reportMetrics(ITestInvocationListener l, TestInfo test,
                        Map<String, String> metrics) {
                    mReportedTestInfo = test;
                    mReportedMetrics = metrics;
                }
            };

            // Startup tests
            mTestInfo = new TestInfo();
            TestInfo t = mTestInfo;  // convenience alias
            t.mTestName = "startup";
            t.mClassName = "com.android.gallery3d.stress.CameraStartUp";
            t.mTestMetricsName = "camera_video_recorder_startup";
            RegexTrie<String> map = t.mPatternMap;
            map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)");
            map.put("CameraStartup", "^Camera average startup time: (\\d+) ms");
            map.put("FirstVideoStartup", "^First Video Startup: (\\d+)");
            map.put("VideoStartup", "^Video average startup time: (\\d+) ms");
        }

        /**
         * Make sure that parsing works in the expected case
         */
        public void testParse() throws Exception {
            String output = join(
                    "First Camera Startup: 1421",  /* "FirstCameraStartup" key */
                    "Camerastartup time: ",
                    "Number of loop: 19",
                    "Individual Camera Startup Time = 1920 ,1929 ,1924 ,1871 ,1840 ,1892 ,1813 " +
                        ",1927 ,1856 ,1929 ,1911 ,1873 ,1381 ,1888 ,2405 ,1926 ,1840 ,2502 " +
                        ",2357 ,",
                    "",
                    "Camera average startup time: 1946 ms",  /* "CameraStartup" key */
                    "",
                    "First Video Startup: 2176",  /* "FirstVideoStartup" key */
                    "Camera Latency : ",
                    "Number of loop: 20",
                    "Avg AutoFocus = 2304",
                    "Avg mShutterLag = 403",
                    "Avg mShutterToPictureDisplayedTime = 369",
                    "Avg mPictureDisplayedToJpegCallbackTime = 50",
                    "Avg mJpegCallbackFinishTime = 1679");

            InputStream iStream = new ByteArrayInputStream(output.getBytes());
            mTestInstance.parseOutputFile(mTestInfo, iStream, null);
            assertEquals(mTestInfo, mReportedTestInfo);
            assertNotNull(mReportedMetrics);
            assertEquals(3, mReportedMetrics.size());
            assertEquals("1946", mReportedMetrics.get("CameraStartup"));
            assertEquals("2176", mReportedMetrics.get("FirstVideoStartup"));
            assertEquals("1421", mReportedMetrics.get("FirstCameraStartup"));
        }
    }
}