| /* |
| * Copyright (C) 2017 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package com.android.media.tests; |
| |
| import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; |
| |
| import com.android.ddmlib.NullOutputReceiver; |
| import com.android.media.tests.AudioLoopbackImageAnalyzer.Result; |
| import com.android.media.tests.AudioLoopbackTestHelper.LogFileType; |
| import com.android.media.tests.AudioLoopbackTestHelper.ResultData; |
| import com.android.tradefed.config.Option; |
| import com.android.tradefed.device.DeviceNotAvailableException; |
| import com.android.tradefed.device.ITestDevice; |
| import com.android.tradefed.log.LogUtil.CLog; |
| import com.android.tradefed.result.FileInputStreamSource; |
| import com.android.tradefed.result.ITestInvocationListener; |
| import com.android.tradefed.result.InputStreamSource; |
| import com.android.tradefed.result.LogDataType; |
| import com.android.tradefed.result.TestDescription; |
| import com.android.tradefed.testtype.IDeviceTest; |
| import com.android.tradefed.testtype.IRemoteTest; |
| import com.android.tradefed.util.FileUtil; |
| import com.android.tradefed.util.RunUtil; |
| |
| import java.io.File; |
| import java.io.IOException; |
| import java.nio.file.Files; |
| import java.util.Arrays; |
| import java.util.Collections; |
| import java.util.Date; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.concurrent.TimeUnit; |
| |
| /** |
| * Runs Audio Latency and Audio Glitch test and reports result. |
| * |
| * <p>Strategy for Audio Latency Stress test: RUN test 1000 times. In each iteration, collect result |
| * files from device, parse and collect data in a ResultData object that also keeps track of |
| * location to test files for a particular iteration. |
| * |
| * <p>ANALYZE test results to produce statistics for 1. Latency and Confidence (Min, Max, Mean, |
| * Median) 2. Create CSV file with test run data 3. Print bad test data to host log file 4. Get |
| * number of test runs with valid data to send to dashboard 5. Produce histogram in host log file; |
| * count number of test results that fall into 1 ms wide buckets. |
| * |
| * <p>UPLOAD test results + log files from “bad” runs; i.e. runs that is missing some or all result |
| * data. |
| */ |
| public class AudioLoopbackTest implements IDeviceTest, IRemoteTest { |
| |
| //=================================================================== |
| // TEST OPTIONS |
| //=================================================================== |
| @Option(name = "run-key", description = "Run key for the test") |
| private String mRunKey = "AudioLoopback"; |
| |
| @Option(name = "sampling-freq", description = "Sampling Frequency for Loopback app") |
| private String mSamplingFreq = "48000"; |
| |
| @Option(name = "mic-source", description = "Mic Source for Loopback app") |
| private String mMicSource = "3"; |
| |
| @Option(name = "audio-thread", description = "Audio Thread for Loopback app") |
| private String mAudioThread = "1"; |
| |
| @Option( |
| name = "audio-level", |
| description = |
| "Audio Level for Loopback app. A device specific" |
| + "param which makes waveform in loopback test hit 60% to 80% range" |
| ) |
| private String mAudioLevel = "-1"; |
| |
| @Option(name = "test-type", description = "Test type to be executed") |
| private String mTestType = TESTTYPE_LATENCY_STR; |
| |
| @Option(name = "buffer-test-duration", description = "Buffer test duration in seconds") |
| private String mBufferTestDuration = "10"; |
| |
| @Option(name = "key-prefix", description = "Key Prefix for reporting") |
| private String mKeyPrefix = "48000_Mic3_"; |
| |
| @Option(name = "iterations", description = "Number of test iterations") |
| private int mIterations = 1; |
| |
| @Option(name = "baseline_latency", description = "") |
| private float mBaselineLatency = 0f; |
| |
| //=================================================================== |
| // CLASS VARIABLES |
| //=================================================================== |
| private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap(); |
| private Map<LogFileType, LogFileData> mFileDataKeyMap; |
| private ITestDevice mDevice; |
| private TestRunHelper mTestRunHelper; |
| private AudioLoopbackTestHelper mLoopbackTestHelper; |
| |
| //=================================================================== |
| // CONSTANTS |
| //=================================================================== |
| private static final String TESTTYPE_LATENCY_STR = "222"; |
| private static final String TESTTYPE_GLITCH_STR = "223"; |
| private static final long TIMEOUT_MS = 5 * 60 * 1000; // 5 min |
| private static final long DEVICE_SYNC_MS = 5 * 60 * 1000; // 5 min |
| private static final long POLLING_INTERVAL_MS = 5 * 1000; |
| private static final int MAX_ATTEMPTS = 3; |
| private static final int MAX_NR_OF_LOG_UPLOADS = 100; |
| |
| private static final int LATENCY_ITERATIONS_LOWER_BOUND = 1; |
| private static final int LATENCY_ITERATIONS_UPPER_BOUND = 10000; |
| private static final int GLITCH_ITERATIONS_LOWER_BOUND = 1; |
| private static final int GLITCH_ITERATIONS_UPPER_BOUND = 1; |
| |
| private static final String DEVICE_TEMP_DIR_PATH = "/sdcard/"; |
| private static final String FMT_OUTPUT_PREFIX = "output_%1$d_" + System.currentTimeMillis(); |
| private static final String FMT_DEVICE_FILENAME = FMT_OUTPUT_PREFIX + "%2$s"; |
| private static final String FMT_DEVICE_PATH = DEVICE_TEMP_DIR_PATH + FMT_DEVICE_FILENAME; |
| |
| private static final String AM_CMD = |
| "am start -n org.drrickorang.loopback/.LoopbackActivity" |
| + " --ei SF %s --es FileName %s --ei MicSource %s --ei AudioThread %s" |
| + " --ei AudioLevel %s --ei TestType %s --ei BufferTestDuration %s"; |
| |
| private static final String ERR_PARAMETER_OUT_OF_BOUNDS = |
| "Test parameter '%1$s' is out of bounds. Lower limit = %2$d, upper limit = %3$d"; |
| |
| private static final String KEY_RESULT_LATENCY_MS = "latency_ms"; |
| private static final String KEY_RESULT_LATENCY_CONFIDENCE = "latency_confidence"; |
| private static final String KEY_RESULT_RECORDER_BENCHMARK = "recorder_benchmark"; |
| private static final String KEY_RESULT_RECORDER_OUTLIER = "recorder_outliers"; |
| private static final String KEY_RESULT_PLAYER_BENCHMARK = "player_benchmark"; |
| private static final String KEY_RESULT_PLAYER_OUTLIER = "player_outliers"; |
| private static final String KEY_RESULT_NUMBER_OF_GLITCHES = "number_of_glitches"; |
| private static final String KEY_RESULT_RECORDER_BUFFER_CALLBACK = "late_recorder_callbacks"; |
| private static final String KEY_RESULT_PLAYER_BUFFER_CALLBACK = "late_player_callbacks"; |
| private static final String KEY_RESULT_GLITCHES_PER_HOUR = "glitches_per_hour"; |
| private static final String KEY_RESULT_TEST_STATUS = "test_status"; |
| private static final String KEY_RESULT_AUDIO_LEVEL = "audio_level"; |
| private static final String KEY_RESULT_RMS = "rms"; |
| private static final String KEY_RESULT_RMS_AVERAGE = "rms_average"; |
| private static final String KEY_RESULT_SAMPLING_FREQUENCY_CONFIDENCE = "sampling_frequency"; |
| private static final String KEY_RESULT_PERIOD_CONFIDENCE = "period_confidence"; |
| private static final String KEY_RESULT_SAMPLING_BLOCK_SIZE = "block_size"; |
| |
| private static final String REDUCED_GLITCHES_TEST_DURATION = "600"; // 10 min |
| |
| private static final LogFileType[] LATENCY_TEST_LOGS = { |
| LogFileType.RESULT, |
| LogFileType.GRAPH, |
| LogFileType.WAVE, |
| LogFileType.PLAYER_BUFFER, |
| LogFileType.PLAYER_BUFFER_HISTOGRAM, |
| LogFileType.PLAYER_BUFFER_PERIOD_TIMES, |
| LogFileType.RECORDER_BUFFER, |
| LogFileType.RECORDER_BUFFER_HISTOGRAM, |
| LogFileType.RECORDER_BUFFER_PERIOD_TIMES, |
| LogFileType.LOGCAT |
| }; |
| |
| private static final LogFileType[] GLITCH_TEST_LOGS = { |
| LogFileType.RESULT, |
| LogFileType.GRAPH, |
| LogFileType.WAVE, |
| LogFileType.PLAYER_BUFFER, |
| LogFileType.PLAYER_BUFFER_HISTOGRAM, |
| LogFileType.PLAYER_BUFFER_PERIOD_TIMES, |
| LogFileType.RECORDER_BUFFER, |
| LogFileType.RECORDER_BUFFER_HISTOGRAM, |
| LogFileType.RECORDER_BUFFER_PERIOD_TIMES, |
| LogFileType.GLITCHES_MILLIS, |
| LogFileType.HEAT_MAP, |
| LogFileType.LOGCAT |
| }; |
| |
| /** |
| * The Audio Latency and Audio Glitch test deals with many various types of log files. To be |
| * able to generate log files in a generic manner, this map is provided to get access to log |
| * file properties like log name prefix, log name file extension and log type (leveraging |
| * tradefed class LogDataType, used when uploading log). |
| */ |
| private final synchronized Map<LogFileType, LogFileData> getLogFileDataKeyMap() { |
| if (mFileDataKeyMap != null) { |
| return mFileDataKeyMap; |
| } |
| |
| final Map<LogFileType, LogFileData> result = new HashMap<LogFileType, LogFileData>(); |
| |
| // Populate dictionary with info about various types of logfiles |
| LogFileData l = new LogFileData(".txt", "result", LogDataType.TEXT); |
| result.put(LogFileType.RESULT, l); |
| |
| l = new LogFileData(".png", "graph", LogDataType.PNG); |
| result.put(LogFileType.GRAPH, l); |
| |
| l = new LogFileData(".wav", "wave", LogDataType.UNKNOWN); |
| result.put(LogFileType.WAVE, l); |
| |
| l = new LogFileData("_playerBufferPeriod.txt", "player_buffer", LogDataType.TEXT); |
| result.put(LogFileType.PLAYER_BUFFER, l); |
| |
| l = new LogFileData("_playerBufferPeriod.png", "player_buffer_histogram", LogDataType.PNG); |
| result.put(LogFileType.PLAYER_BUFFER_HISTOGRAM, l); |
| |
| String fileExtension = "_playerBufferPeriodTimes.txt"; |
| String uploadName = "player_buffer_period_times"; |
| l = new LogFileData(fileExtension, uploadName, LogDataType.TEXT); |
| result.put(LogFileType.PLAYER_BUFFER_PERIOD_TIMES, l); |
| |
| l = new LogFileData("_recorderBufferPeriod.txt", "recorder_buffer", LogDataType.TEXT); |
| result.put(LogFileType.RECORDER_BUFFER, l); |
| |
| fileExtension = "_recorderBufferPeriod.png"; |
| uploadName = "recorder_buffer_histogram"; |
| l = new LogFileData(fileExtension, uploadName, LogDataType.PNG); |
| result.put(LogFileType.RECORDER_BUFFER_HISTOGRAM, l); |
| |
| fileExtension = "_recorderBufferPeriodTimes.txt"; |
| uploadName = "recorder_buffer_period_times"; |
| l = new LogFileData(fileExtension, uploadName, LogDataType.TEXT); |
| result.put(LogFileType.RECORDER_BUFFER_PERIOD_TIMES, l); |
| |
| l = new LogFileData("_glitchMillis.txt", "glitches_millis", LogDataType.TEXT); |
| result.put(LogFileType.GLITCHES_MILLIS, l); |
| |
| |
| l = new LogFileData("_heatMap.png", "heat_map", LogDataType.PNG); |
| result.put(LogFileType.HEAT_MAP, l); |
| |
| l = new LogFileData(".txt", "logcat", LogDataType.TEXT); |
| result.put(LogFileType.LOGCAT, l); |
| |
| mFileDataKeyMap = Collections.unmodifiableMap(result); |
| return mFileDataKeyMap; |
| } |
| |
| private static final Map<String, String> createMetricsKeyMap() { |
| final Map<String, String> result = new HashMap<String, String>(); |
| |
| result.put("LatencyMs", KEY_RESULT_LATENCY_MS); |
| result.put("LatencyConfidence", KEY_RESULT_LATENCY_CONFIDENCE); |
| result.put("SF", KEY_RESULT_SAMPLING_FREQUENCY_CONFIDENCE); |
| result.put("Recorder Benchmark", KEY_RESULT_RECORDER_BENCHMARK); |
| result.put("Recorder Number of Outliers", KEY_RESULT_RECORDER_OUTLIER); |
| result.put("Player Benchmark", KEY_RESULT_PLAYER_BENCHMARK); |
| result.put("Player Number of Outliers", KEY_RESULT_PLAYER_OUTLIER); |
| result.put("Total Number of Glitches", KEY_RESULT_NUMBER_OF_GLITCHES); |
| result.put("kth% Late Recorder Buffer Callbacks", KEY_RESULT_RECORDER_BUFFER_CALLBACK); |
| result.put("kth% Late Player Buffer Callbacks", KEY_RESULT_PLAYER_BUFFER_CALLBACK); |
| result.put("Glitches Per Hour", KEY_RESULT_GLITCHES_PER_HOUR); |
| result.put("Test Status", KEY_RESULT_TEST_STATUS); |
| result.put("AudioLevel", KEY_RESULT_AUDIO_LEVEL); |
| result.put("RMS", KEY_RESULT_RMS); |
| result.put("Average", KEY_RESULT_RMS_AVERAGE); |
| result.put("PeriodConfidence", KEY_RESULT_PERIOD_CONFIDENCE); |
| result.put("BS", KEY_RESULT_SAMPLING_BLOCK_SIZE); |
| |
| return Collections.unmodifiableMap(result); |
| } |
| |
| //=================================================================== |
| // ENUMS |
| //=================================================================== |
| public enum TestType { |
| GLITCH, |
| LATENCY, |
| LATENCY_STRESS, |
| NONE |
| } |
| |
| //=================================================================== |
| // INNER CLASSES |
| //=================================================================== |
| public final class LogFileData { |
| private String fileExtension; |
| private String filePrefix; |
| private LogDataType logDataType; |
| |
| private LogFileData(String fileExtension, String filePrefix, LogDataType logDataType) { |
| this.fileExtension = fileExtension; |
| this.filePrefix = filePrefix; |
| this.logDataType = logDataType; |
| } |
| } |
| |
| //=================================================================== |
| // FUNCTIONS |
| //=================================================================== |
| |
| /** {@inheritDoc} */ |
| @Override |
| public void setDevice(ITestDevice device) { |
| mDevice = device; |
| } |
| |
| /** {@inheritDoc} */ |
| @Override |
| public ITestDevice getDevice() { |
| return mDevice; |
| } |
| |
| /** |
| * Test Entry Point |
| * |
| * <p>{@inheritDoc} |
| */ |
| @Override |
| public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { |
| |
| initializeTest(listener); |
| |
| mTestRunHelper.startTest(1); |
| |
| Map<String, String> metrics = null; |
| try { |
| if (!verifyTestParameters()) { |
| return; |
| } |
| |
| // Stop logcat logging so we can record one logcat log per iteration |
| getDevice().stopLogcat(); |
| |
| switch (getTestType()) { |
| case GLITCH: |
| runGlitchesTest(mTestRunHelper, mLoopbackTestHelper); |
| break; |
| case LATENCY: |
| case LATENCY_STRESS: |
| // Run test iterations |
| runLatencyTest(mLoopbackTestHelper, mIterations); |
| break; |
| default: |
| break; |
| } |
| |
| mLoopbackTestHelper.processTestData(); |
| metrics = uploadLogsReturnMetrics(listener); |
| CLog.i("Uploading metrics values:\n" + Arrays.toString(metrics.entrySet().toArray())); |
| mTestRunHelper.endTest(metrics); |
| } catch (TestFailureException e) { |
| CLog.i("TestRunHelper.reportFailure triggered"); |
| } finally { |
| CLog.i("Test ended - cleanup"); |
| deleteAllTempFiles(); |
| getDevice().startLogcat(); |
| } |
| } |
| |
| private void runLatencyTest(AudioLoopbackTestHelper loopbackTestHelper, int iterations) |
| throws DeviceNotAvailableException, TestFailureException { |
| for (int i = 0; i < iterations; i++) { |
| CLog.i("---- Iteration " + i + " of " + (iterations - 1) + " -----"); |
| |
| final ResultData d = new ResultData(); |
| d.setIteration(i); |
| Map<String, String> resultsDictionary = null; |
| resultsDictionary = runTest(d, getSingleTestTimeoutValue()); |
| loopbackTestHelper.addTestData(d, resultsDictionary, true); |
| } |
| } |
| |
| /** |
| * Glitches test, strategy: |
| * <p> |
| * |
| * <ul> |
| * <li>1. Calibrate Audio level |
| * <li>2. Run Audio Latency test until seeing good waveform |
| * <li>3. Run small Glitches test, 5-10 seconds |
| * <li>4. If numbers look good, run long Glitches test, else run reduced Glitches test |
| * </ul> |
| * |
| * @param testRunHelper |
| * @param loopbackTestHelper |
| * @throws DeviceNotAvailableException |
| * @throws TestFailureException |
| */ |
| private void runGlitchesTest(TestRunHelper testRunHelper, |
| AudioLoopbackTestHelper loopbackTestHelper) |
| throws DeviceNotAvailableException, TestFailureException { |
| final int MAX_RETRIES = 3; |
| int nrOfSuccessfulTests; |
| int counter = 0; |
| AudioLoopbackTestHelper tempTestHelper = null; |
| boolean runningReducedGlitchesTest = false; |
| |
| // Step 1: Calibrate Audio level |
| // Step 2: Run Audio Latency test until seeing good waveform |
| final int LOOPBACK_ITERATIONS = 4; |
| final String originalTestType = mTestType; |
| final String originalBufferTestDuration = mBufferTestDuration; |
| mTestType = TESTTYPE_LATENCY_STR; |
| do { |
| nrOfSuccessfulTests = 0; |
| tempTestHelper = new AudioLoopbackTestHelper(LOOPBACK_ITERATIONS); |
| runLatencyTest(tempTestHelper, LOOPBACK_ITERATIONS); |
| nrOfSuccessfulTests = tempTestHelper.processTestData(); |
| counter++; |
| } while (nrOfSuccessfulTests <= 0 && counter <= MAX_RETRIES); |
| |
| if (nrOfSuccessfulTests <= 0) { |
| testRunHelper.reportFailure("Glitch Setup failed: Latency test"); |
| } |
| |
| // Retrieve audio level from successful test |
| int audioLevel = -1; |
| List<ResultData> results = tempTestHelper.getAllTestData(); |
| for (ResultData rd : results) { |
| // Check if test passed |
| if (rd.getImageAnalyzerResult() == Result.PASS && rd.getConfidence() == 1.0) { |
| audioLevel = rd.getAudioLevel(); |
| break; |
| } |
| } |
| |
| if (audioLevel < 6) { |
| testRunHelper.reportFailure("Glitch Setup failed: Audio level not valid"); |
| } |
| |
| CLog.i("Audio Glitch: Audio level is " + audioLevel); |
| |
| // Step 3: Run small Glitches test, 5-10 seconds |
| mTestType = originalTestType; |
| mBufferTestDuration = "10"; |
| mAudioLevel = Integer.toString(audioLevel); |
| |
| counter = 0; |
| int glitches = -1; |
| do { |
| tempTestHelper = new AudioLoopbackTestHelper(1); |
| runLatencyTest(tempTestHelper, 1); |
| Map<String, String> resultsDictionary = |
| tempTestHelper.getResultDictionaryForIteration(0); |
| final String nrOfGlitches = |
| resultsDictionary.get(getMetricsKey(KEY_RESULT_NUMBER_OF_GLITCHES)); |
| glitches = Integer.parseInt(nrOfGlitches); |
| CLog.i("10 s glitch test produced " + glitches + " glitches"); |
| counter++; |
| } while (glitches > 10 || glitches < 0 && counter <= MAX_RETRIES); |
| |
| // Step 4: If numbers look good, run long Glitches test |
| if (glitches > 10 || glitches < 0) { |
| // Reduce test time and set some values to 0 once test completes |
| runningReducedGlitchesTest = true; |
| mBufferTestDuration = REDUCED_GLITCHES_TEST_DURATION; |
| } else { |
| mBufferTestDuration = originalBufferTestDuration; |
| } |
| |
| final ResultData d = new ResultData(); |
| d.setIteration(0); |
| Map<String, String> resultsDictionary = null; |
| resultsDictionary = runTest(d, getSingleTestTimeoutValue()); |
| if (runningReducedGlitchesTest) { |
| // Special treatment, we want to upload values, but also indicate that pre-test |
| // conditions failed. We will set the glitches count and zero out the rest. |
| String[] testValuesToChangeArray = new String[] { |
| KEY_RESULT_RECORDER_BENCHMARK, |
| KEY_RESULT_RECORDER_OUTLIER, |
| KEY_RESULT_PLAYER_BENCHMARK, |
| KEY_RESULT_PLAYER_OUTLIER, |
| KEY_RESULT_RECORDER_BUFFER_CALLBACK, |
| KEY_RESULT_PLAYER_BUFFER_CALLBACK |
| }; |
| |
| for (String key : testValuesToChangeArray) { |
| final String metricsKey = getMetricsKey(key); |
| if (resultsDictionary.containsKey(metricsKey)) { |
| resultsDictionary.put(metricsKey, "0"); |
| } |
| } |
| } |
| |
| loopbackTestHelper.addTestData(d, resultsDictionary, false); |
| } |
| |
| private void initializeTest(ITestInvocationListener listener) |
| throws UnsupportedOperationException, DeviceNotAvailableException { |
| |
| mFileDataKeyMap = getLogFileDataKeyMap(); |
| TestDescription testId = new TestDescription(getClass().getCanonicalName(), mRunKey); |
| |
| // Allocate helpers |
| mTestRunHelper = new TestRunHelper(listener, testId); |
| mLoopbackTestHelper = new AudioLoopbackTestHelper(mIterations); |
| |
| getDevice().disableKeyguard(); |
| getDevice().waitForDeviceAvailable(DEVICE_SYNC_MS); |
| |
| getDevice().setDate(new Date()); |
| CLog.i("syncing device time to host time"); |
| } |
| |
| private Map<String, String> runTest(ResultData data, final long timeout) |
| throws DeviceNotAvailableException, TestFailureException { |
| |
| // start measurement and wait for result file |
| final NullOutputReceiver receiver = new NullOutputReceiver(); |
| |
| final String loopbackCmd = getTestCommand(data.getIteration()); |
| CLog.i("Loopback cmd: " + loopbackCmd); |
| |
| // Clear logcat |
| // Seems like getDevice().clearLogcat(); doesn't do anything? |
| // Do it through ADB |
| getDevice().executeAdbCommand("logcat", "-c"); |
| final long deviceTestStartTime = getDevice().getDeviceDate(); |
| |
| getDevice() |
| .executeShellCommand( |
| loopbackCmd, receiver, TIMEOUT_MS, TimeUnit.MILLISECONDS, MAX_ATTEMPTS); |
| |
| final long loopbackStartTime = System.currentTimeMillis(); |
| File loopbackReport = null; |
| |
| data.setDeviceTestStartTime(deviceTestStartTime); |
| |
| // Try to retrieve result file from device. |
| final String resultFilename = getDeviceFilename(LogFileType.RESULT, data.getIteration()); |
| do { |
| RunUtil.getDefault().sleep(POLLING_INTERVAL_MS); |
| if (getDevice().doesFileExist(resultFilename)) { |
| // Store device log files in tmp directory on Host and add to ResultData object |
| storeDeviceFilesOnHost(data); |
| final String reportFilename = data.getLogFile(LogFileType.RESULT); |
| if (reportFilename != null && !reportFilename.isEmpty()) { |
| loopbackReport = new File(reportFilename); |
| if (loopbackReport.length() > 0) { |
| break; |
| } |
| } |
| } |
| |
| data.setIsTimedOut(System.currentTimeMillis() - loopbackStartTime >= timeout); |
| } while (!data.hasLogFile(LogFileType.RESULT) && !data.isTimedOut()); |
| |
| // Grab logcat for iteration |
| try (final InputStreamSource lc = getDevice().getLogcatSince(deviceTestStartTime)) { |
| saveLogcatForIteration(data, lc, data.getIteration()); |
| } |
| |
| // Check if test timed out. If so, don't fail the test, but return to upper logic. |
| // We accept certain number of individual test timeouts. |
| if (data.isTimedOut()) { |
| // No device result files retrieved, so no need to parse |
| return null; |
| } |
| |
| // parse result |
| Map<String, String> loopbackResult = null; |
| |
| try { |
| loopbackResult = |
| AudioLoopbackTestHelper.parseKeyValuePairFromFile( |
| loopbackReport, METRICS_KEY_MAP, mKeyPrefix, "=", "%s: %s"); |
| populateResultData(loopbackResult, data); |
| |
| // Trust but verify, so get Audio Level from ADB and compare to value from app |
| final int adbAudioLevel = |
| AudioLevelUtility.extractDeviceHeadsetLevelFromAdbShell(getDevice()); |
| if (adbAudioLevel > -1 && data.getAudioLevel() != adbAudioLevel) { |
| final String errMsg = |
| String.format( |
| "App Audio Level (%1$d) differs from ADB level (%2$d)", |
| data.getAudioLevel(), adbAudioLevel); |
| mTestRunHelper.reportFailure(errMsg); |
| } |
| } catch (final IOException ioe) { |
| CLog.e(ioe); |
| mTestRunHelper.reportFailure("I/O error while parsing Loopback result."); |
| } catch (final NumberFormatException ne) { |
| CLog.e(ne); |
| mTestRunHelper.reportFailure("Number format error parsing Loopback result."); |
| } |
| |
| return loopbackResult; |
| } |
| |
| private String getMetricsKey(final String key) { |
| return mKeyPrefix + key; |
| } |
| private final long getSingleTestTimeoutValue() { |
| return Long.parseLong(mBufferTestDuration) * 1000 + TIMEOUT_MS; |
| } |
| |
| private Map<String, String> uploadLogsReturnMetrics(ITestInvocationListener listener) |
| throws DeviceNotAvailableException, TestFailureException { |
| |
| // "resultDictionary" is used to post results to dashboards like BlackBox |
| // "results" contains test logs to be uploaded; i.e. to Sponge |
| |
| List<ResultData> results = null; |
| Map<String, String> resultDictionary = new HashMap<String, String>(); |
| |
| switch (getTestType()) { |
| case GLITCH: |
| resultDictionary = mLoopbackTestHelper.getResultDictionaryForIteration(0); |
| // Upload all test files to be backward compatible with old test |
| results = mLoopbackTestHelper.getAllTestData(); |
| break; |
| case LATENCY: |
| { |
| final int nrOfValidResults = mLoopbackTestHelper.processTestData(); |
| if (nrOfValidResults == 0) { |
| mTestRunHelper.reportFailure("No good data was collected"); |
| } else { |
| // use dictionary collected from single test run |
| resultDictionary = mLoopbackTestHelper.getResultDictionaryForIteration(0); |
| } |
| |
| // Upload all test files to be backward compatible with old test |
| results = mLoopbackTestHelper.getAllTestData(); |
| } |
| break; |
| case LATENCY_STRESS: |
| { |
| final int nrOfValidResults = mLoopbackTestHelper.processTestData(); |
| if (nrOfValidResults == 0) { |
| mTestRunHelper.reportFailure("No good data was collected"); |
| } else { |
| mLoopbackTestHelper.populateStressTestMetrics(resultDictionary, mKeyPrefix); |
| } |
| |
| results = mLoopbackTestHelper.getWorstResults(MAX_NR_OF_LOG_UPLOADS); |
| |
| // Save all test data in a spreadsheet style csv file for post test analysis |
| try { |
| saveResultsAsCSVFile(listener); |
| } catch (final IOException e) { |
| CLog.e(e); |
| } |
| } |
| break; |
| default: |
| break; |
| } |
| |
| // Upload relevant logs |
| for (final ResultData d : results) { |
| final LogFileType[] logFileTypes = getLogFileTypesForCurrentTest(); |
| for (final LogFileType logType : logFileTypes) { |
| uploadLog(listener, logType, d); |
| } |
| } |
| |
| return resultDictionary; |
| } |
| |
| private TestType getTestType() { |
| if (mTestType.equals(TESTTYPE_GLITCH_STR)) { |
| if (GLITCH_ITERATIONS_LOWER_BOUND <= mIterations |
| && mIterations <= GLITCH_ITERATIONS_UPPER_BOUND) { |
| return TestType.GLITCH; |
| } |
| } |
| |
| if (mTestType.equals(TESTTYPE_LATENCY_STR)) { |
| if (mIterations == 1) { |
| return TestType.LATENCY; |
| } |
| |
| if (LATENCY_ITERATIONS_LOWER_BOUND <= mIterations |
| && mIterations <= LATENCY_ITERATIONS_UPPER_BOUND) { |
| return TestType.LATENCY_STRESS; |
| } |
| } |
| |
| return TestType.NONE; |
| } |
| |
| private boolean verifyTestParameters() throws TestFailureException { |
| if (getTestType() != TestType.NONE) { |
| return true; |
| } |
| |
| if (mTestType.equals(TESTTYPE_GLITCH_STR) |
| && (mIterations < GLITCH_ITERATIONS_LOWER_BOUND |
| || mIterations > GLITCH_ITERATIONS_UPPER_BOUND)) { |
| final String error = |
| String.format( |
| ERR_PARAMETER_OUT_OF_BOUNDS, |
| "iterations", |
| GLITCH_ITERATIONS_LOWER_BOUND, |
| GLITCH_ITERATIONS_UPPER_BOUND); |
| mTestRunHelper.reportFailure(error); |
| return false; |
| } |
| |
| if (mTestType.equals(TESTTYPE_LATENCY_STR) |
| && (mIterations < LATENCY_ITERATIONS_LOWER_BOUND |
| || mIterations > LATENCY_ITERATIONS_UPPER_BOUND)) { |
| final String error = |
| String.format( |
| ERR_PARAMETER_OUT_OF_BOUNDS, |
| "iterations", |
| LATENCY_ITERATIONS_LOWER_BOUND, |
| LATENCY_ITERATIONS_UPPER_BOUND); |
| mTestRunHelper.reportFailure(error); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| private void populateResultData(final Map<String, String> results, ResultData data) { |
| if (results == null || results.isEmpty()) { |
| return; |
| } |
| |
| String key = getMetricsKey(KEY_RESULT_LATENCY_MS); |
| if (results.containsKey(key)) { |
| data.setLatency(Float.parseFloat(results.get(key))); |
| } |
| |
| key = getMetricsKey(KEY_RESULT_LATENCY_CONFIDENCE); |
| if (results.containsKey(key)) { |
| data.setConfidence(Float.parseFloat(results.get(key))); |
| } |
| |
| key = getMetricsKey(KEY_RESULT_AUDIO_LEVEL); |
| if (results.containsKey(key)) { |
| data.setAudioLevel(Integer.parseInt(results.get(key))); |
| } |
| |
| key = getMetricsKey(KEY_RESULT_RMS); |
| if (results.containsKey(key)) { |
| data.setRMS(Float.parseFloat(results.get(key))); |
| } |
| |
| key = getMetricsKey(KEY_RESULT_RMS_AVERAGE); |
| if (results.containsKey(key)) { |
| data.setRMSAverage(Float.parseFloat(results.get(key))); |
| } |
| |
| key = getMetricsKey(KEY_RESULT_PERIOD_CONFIDENCE); |
| if (results.containsKey(key)) { |
| data.setPeriodConfidence(Float.parseFloat(results.get(key))); |
| } |
| |
| key = getMetricsKey(KEY_RESULT_SAMPLING_BLOCK_SIZE); |
| if (results.containsKey(key)) { |
| data.setBlockSize(Integer.parseInt(results.get(key))); |
| } |
| } |
| |
| private void storeDeviceFilesOnHost(ResultData data) throws DeviceNotAvailableException { |
| final int iteration = data.getIteration(); |
| for (final LogFileType log : getLogFileTypesForCurrentTest()) { |
| if (getDevice().doesFileExist(getDeviceFilename(log, iteration))) { |
| final String deviceFileName = getDeviceFilename(log, iteration); |
| final File logFile = getDevice().pullFile(deviceFileName); |
| data.setLogFile(log, logFile.getAbsolutePath()); |
| CLog.i("Delete file from device: " + deviceFileName); |
| deleteFileFromDevice(deviceFileName); |
| } |
| } |
| } |
| |
| private void deleteAllTempFiles() { |
| for (final ResultData d : mLoopbackTestHelper.getAllTestData()) { |
| final LogFileType[] logFileTypes = getLogFileTypesForCurrentTest(); |
| for (final LogFileType logType : logFileTypes) { |
| final String logFilename = d.getLogFile(logType); |
| if (logFilename == null || logFilename.isEmpty()) { |
| CLog.e("Logfile not found for LogFileType=" + logType.name()); |
| } else { |
| FileUtil.deleteFile(new File(logFilename)); |
| } |
| } |
| } |
| } |
| |
| private void deleteFileFromDevice(String deviceFileName) throws DeviceNotAvailableException { |
| getDevice().executeShellCommand("rm -f " + deviceFileName); |
| } |
| |
| private final LogFileType[] getLogFileTypesForCurrentTest() { |
| switch (getTestType()) { |
| case GLITCH: |
| return GLITCH_TEST_LOGS; |
| case LATENCY: |
| case LATENCY_STRESS: |
| return LATENCY_TEST_LOGS; |
| default: |
| return null; |
| } |
| } |
| |
| private String getKeyPrefixForIteration(int iteration) { |
| if (mIterations == 1) { |
| // If only one run, skip the iteration number |
| return mKeyPrefix; |
| } |
| return mKeyPrefix + iteration + "_"; |
| } |
| |
| private String getDeviceFilename(LogFileType key, int iteration) { |
| final Map<LogFileType, LogFileData> map = getLogFileDataKeyMap(); |
| if (map.containsKey(key)) { |
| final LogFileData data = map.get(key); |
| return String.format(FMT_DEVICE_PATH, iteration, data.fileExtension); |
| } |
| return null; |
| } |
| |
| private void uploadLog(ITestInvocationListener listener, LogFileType key, ResultData data) { |
| final Map<LogFileType, LogFileData> map = getLogFileDataKeyMap(); |
| if (!map.containsKey(key)) { |
| return; |
| } |
| |
| final LogFileData logInfo = map.get(key); |
| final String prefix = getKeyPrefixForIteration(data.getIteration()) + logInfo.filePrefix; |
| final LogDataType logDataType = logInfo.logDataType; |
| final String logFilename = data.getLogFile(key); |
| if (logFilename == null || logFilename.isEmpty()) { |
| CLog.e("Logfile not found for LogFileType=" + key.name()); |
| } else { |
| File logFile = new File(logFilename); |
| try (InputStreamSource iss = new FileInputStreamSource(logFile)) { |
| listener.testLog(prefix, logDataType, iss); |
| } |
| } |
| } |
| |
| private void saveLogcatForIteration(ResultData data, InputStreamSource logcat, int iteration) { |
| if (logcat == null) { |
| CLog.i("Logcat could not be saved for iteration " + iteration); |
| return; |
| } |
| |
| //create a temp file |
| File temp; |
| try { |
| temp = FileUtil.createTempFile("logcat_" + iteration + "_", ".txt"); |
| data.setLogFile(LogFileType.LOGCAT, temp.getAbsolutePath()); |
| |
| // Copy logcat data into temp file |
| Files.copy(logcat.createInputStream(), temp.toPath(), REPLACE_EXISTING); |
| logcat.close(); |
| } catch (final IOException e) { |
| CLog.i("Error when saving logcat for iteration=" + iteration); |
| CLog.e(e); |
| } |
| } |
| |
| private void saveResultsAsCSVFile(ITestInvocationListener listener) |
| throws DeviceNotAvailableException, IOException { |
| final File csvTmpFile = File.createTempFile("audio_test_data", "csv"); |
| mLoopbackTestHelper.writeAllResultsToCSVFile(csvTmpFile, getDevice()); |
| try (InputStreamSource iss = new FileInputStreamSource(csvTmpFile)) { |
| listener.testLog("audio_test_data", LogDataType.JACOCO_CSV, iss); |
| } |
| // cleanup |
| csvTmpFile.delete(); |
| } |
| |
| private String getTestCommand(int currentIteration) { |
| return String.format( |
| AM_CMD, |
| mSamplingFreq, |
| String.format(FMT_OUTPUT_PREFIX, currentIteration), |
| mMicSource, |
| mAudioThread, |
| mAudioLevel, |
| mTestType, |
| mBufferTestDuration); |
| } |
| } |
