CtsMediaBitstreamsTestCases: refactor metric entry format
Log test results in the following format for simpler data analysis:
{
"path": "path/to/bitstream",
"codec_name": "omx.decoder.name",
"status": "unsupported|true|false|crash|timeout"
}
Bug: 30268664
Test: cts-tradefed run cts -m CtsMediaBitstreamsTestCases
Change-Id: I3d6c38dd50f839d422e5a11ec523958889fcd5bf
diff --git a/common/device-side/util/src/com/android/compatibility/common/util/MediaUtils.java b/common/device-side/util/src/com/android/compatibility/common/util/MediaUtils.java
index 1727a3a..c7d2fa5 100644
--- a/common/device-side/util/src/com/android/compatibility/common/util/MediaUtils.java
+++ b/common/device-side/util/src/com/android/compatibility/common/util/MediaUtils.java
@@ -245,6 +245,18 @@
return getCodecNames(true /* isEncoder */, null /* isGoog */, formats);
}
+ public static String[] getDecoderNamesForMime(String mime) {
+ MediaFormat format = new MediaFormat();
+ format.setString(MediaFormat.KEY_MIME, mime);
+ return getCodecNames(false /* isEncoder */, null /* isGoog */, format);
+ }
+
+ public static String[] getEncoderNamesForMime(String mime) {
+ MediaFormat format = new MediaFormat();
+ format.setString(MediaFormat.KEY_MIME, mime);
+ return getCodecNames(true /* isEncoder */, null /* isGoog */, format);
+ }
+
public static void verifyNumCodecs(
int count, boolean isEncoder, Boolean isGoog, MediaFormat... formats) {
String desc = (isEncoder ? "encoders" : "decoders") + " for "
diff --git a/hostsidetests/media/bitstreams/app/src/android/media/cts/bitstreams/app/MediaBitstreamsDeviceSideTest.java b/hostsidetests/media/bitstreams/app/src/android/media/cts/bitstreams/app/MediaBitstreamsDeviceSideTest.java
index 92f0d2a..32fc86b 100644
--- a/hostsidetests/media/bitstreams/app/src/android/media/cts/bitstreams/app/MediaBitstreamsDeviceSideTest.java
+++ b/hostsidetests/media/bitstreams/app/src/android/media/cts/bitstreams/app/MediaBitstreamsDeviceSideTest.java
@@ -21,6 +21,7 @@
import android.content.SharedPreferences;
import android.content.SharedPreferences.Editor;
import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.media.cts.bitstreams.MediaBitstreams;
@@ -161,12 +162,18 @@
}
MediaFormat format = parseTrackFormat(formatStr);
- if (!MediaUtils.checkDecoderForFormat(format)) {
- continue;
- }
+ String mime = format.getString(MediaFormat.KEY_MIME);
+ String[] decoders = MediaUtils.getDecoderNamesForMime(mime);
ps.println(path);
+ ps.println(decoders.length);
+ for (String name : decoders) {
+ ps.println(name);
+ ps.println(MediaUtils.supports(name, format));
+ }
+
}
+
ps.flush();
}
}
diff --git a/hostsidetests/media/bitstreams/common/src/android/media/cts/bitstreams/MediaBitstreams.java b/hostsidetests/media/bitstreams/common/src/android/media/cts/bitstreams/MediaBitstreams.java
index 8b9a507..b4ab76c 100644
--- a/hostsidetests/media/bitstreams/common/src/android/media/cts/bitstreams/MediaBitstreams.java
+++ b/hostsidetests/media/bitstreams/common/src/android/media/cts/bitstreams/MediaBitstreams.java
@@ -38,17 +38,21 @@
public static final String DEFAULT_DEVICE_BITSTEAMS_PATH = "/data/local/tmp/TestVectorsIttiam";
/* metric keys */
- public static final String KEY_BITSTREAMS_FORMATS_XML = "bitstreams-formats-xml";
- public static final String KEY_SUPPORTED_BITSTREAMS_TXT = "supported-bitstreams-txt";
- public static final String KEY_BITSTREAMS_VALIDATION_TXT = "bitstreams-validation-txt";
- public static final String KEY_APP_CACHE_DIR = "app-cache-dir";
- public static final String KEY_ERR_MSG = "err-msg";
+ public static final String KEY_BITSTREAMS_FORMATS_XML = "bitstreams_formats_xml";
+ public static final String KEY_SUPPORTED_BITSTREAMS_TXT = "supported_bitstreams_txt";
+ public static final String KEY_BITSTREAMS_VALIDATION_TXT = "bitstreams_validation_txt";
+ public static final String KEY_APP_CACHE_DIR = "app_cache_dir";
+ public static final String KEY_ERR_MSG = "err_msg";
+ public static final String KEY_PATH = "path";
+ public static final String KEY_CODEC_NAME = "codec_name";
+ public static final String KEY_STATUS = "status";
/* constants */
public static final String K_MODULE = "CtsMediaBitstreamsTestCases";
public static final String K_BITSTREAMS_LIST_TXT = "bitstreamsFile.txt";
public static final String K_TEST_GET_SUPPORTED_BITSTREAMS = "testGetSupportedBitstreams";
public static final String K_NATIVE_CRASH = "native crash";
+ public static final String K_UNSUPPORTED = "unsupported";
public static final String DYNAMIC_CONFIG_XML = "DynamicConfig.xml";
public static final String DYNAMIC_CONFIG = "dynamicConfig";
diff --git a/hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/MediaBitstreamsTest.java b/hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/MediaBitstreamsTest.java
index a82b9c3..0da0193 100644
--- a/hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/MediaBitstreamsTest.java
+++ b/hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/MediaBitstreamsTest.java
@@ -46,6 +46,7 @@
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.junit.Ignore;
import org.junit.Test;
@@ -168,6 +169,29 @@
return Thread.currentThread().getStackTrace()[2].getMethodName();
}
+ private MetricsReportLog createReport(String methodName) {
+ String className = MediaBitstreamsTest.class.getCanonicalName();
+ MetricsReportLog report = new MetricsReportLog(
+ mBuildHelper.getBuildInfo(), mAbi.getName(),
+ String.format("%s#%s", className, methodName),
+ MediaBitstreams.K_MODULE, "media_bitstreams_conformance");
+ return report;
+ }
+
+ /**
+ * @param method test method name in the form class#method
+ * @param p path to bitstream
+ * @param d decoder name
+ * @param s test status: unsupported, true, false, crash, or timeout.
+ */
+ private void addConformanceEntry(String method, String p, String d, String s) {
+ MetricsReportLog report = createReport(method);
+ report.addValue(MediaBitstreams.KEY_PATH, p, ResultType.NEUTRAL, ResultUnit.NONE);
+ report.addValue(MediaBitstreams.KEY_CODEC_NAME, d, ResultType.NEUTRAL, ResultUnit.NONE);
+ report.addValue(MediaBitstreams.KEY_STATUS, s, ResultType.NEUTRAL, ResultUnit.NONE);
+ report.submit();
+ }
+
Map<String, String> getArgs() {
Map<String, String> args = new HashMap<>();
args.put(MediaBitstreams.OPT_DEBUG_TARGET_DEVICE, Boolean.toString(mDebugTargetDevice));
@@ -274,14 +298,9 @@
String path = lines[i++];
mProcessedBitstreams.add(path);
- String className = MediaBitstreamsTest.class.getCanonicalName();
- MetricsReportLog report = new MetricsReportLog(
- mBuildHelper.getBuildInfo(), mAbi.getName(),
- String.format("%s#%s", className, mMethodName),
- getClass().getSimpleName(), path.replaceAll("[./]", "_"));
+ String errMsg;
boolean failedEarly;
- String errMsg;
if (i < lines.length) {
failedEarly = Boolean.parseBoolean(lines[i++]);
errMsg = failedEarly ? lines[i++] : "";
@@ -291,27 +310,26 @@
mLastCrash = MediaBitstreams.generateCrashSignature(path, "");
mProcessedBitstreams.removeLast();
}
+
if (failedEarly) {
- String keyErrMsg = MediaBitstreams.KEY_ERR_MSG;
- report.addValue(keyErrMsg, errMsg, ResultType.NEUTRAL, ResultUnit.NONE);
- report.submit();
+ addConformanceEntry(mMethodName, path, null, errMsg);
continue;
}
int n = Integer.parseInt(lines[i++]);
for (int j = 0; j < n && i < lines.length; j++) {
- String name = lines[i++];
+ String decoderName = lines[i++];
String result;
if (i < lines.length) {
result = lines[i++];
} else {
result = MediaBitstreams.K_NATIVE_CRASH;
- mLastCrash = MediaBitstreams.generateCrashSignature(path, name);
+ mLastCrash = MediaBitstreams.generateCrashSignature(path, decoderName);
mProcessedBitstreams.removeLast();
}
- report.addValue(name, result, ResultType.NEUTRAL, ResultUnit.NONE);
+ addConformanceEntry(mMethodName, path, decoderName, result);
}
- report.submit();
+
}
}
@@ -365,6 +383,7 @@
device,
MediaBitstreams.K_TEST_GET_SUPPORTED_BITSTREAMS,
MediaBitstreams.KEY_SUPPORTED_BITSTREAMS_TXT);
+ Set<String> bitstreams = preparer.getBitstreams();
Set<String> supportedBitstreams = preparer.getSupportedBitstreams();
CLog.i("%d supported bitstreams under %s", supportedBitstreams.size(), prefix);
@@ -372,35 +391,43 @@
long size = 0;
long limit = device.getExternalStoreFreeSpace() * mUtilizationRate * 1024 / 100;
- String currentMethod = getCurrentMethod();
- Set<String> bitstreams = new LinkedHashSet<>();
- Iterator<String> iter = supportedBitstreams.iterator();
+ String curMethod = getCurrentMethod();
+ Set<String> toPush = new LinkedHashSet<>();
+ Iterator<String> iter = bitstreams.iterator();
- for (int i = 0; i < supportedBitstreams.size(); i++) {
+ for (int i = 0; i < bitstreams.size(); i++) {
if (n >= mNumBatches) {
break;
}
- String bitstreamPath = iter.next();
- File bitstreamFile = new File(mHostBitstreamsPath, bitstreamPath);
- String md5Path = MediaBitstreams.getMd5Path(bitstreamPath);
- File md5File = new File(mHostBitstreamsPath, md5Path);
-
- if (md5File.exists() && bitstreamFile.exists()) {
- size += md5File.length();
- size += bitstreamFile.length();
- bitstreams.add(bitstreamPath);
+ String p = iter.next();
+ Map<String, Boolean> decoderCapabilities;
+ decoderCapabilities = preparer.getDecoderCapabilitiesForPath(p);
+ for (Entry<String, Boolean> entry : decoderCapabilities.entrySet()) {
+ Boolean supported = entry.getValue();
+ if (supported) {
+ File bitstreamFile = new File(mHostBitstreamsPath, p);
+ String md5Path = MediaBitstreams.getMd5Path(p);
+ File md5File = new File(mHostBitstreamsPath, md5Path);
+ if (md5File.exists() && bitstreamFile.exists() && toPush.add(p)) {
+ size += md5File.length();
+ size += bitstreamFile.length();
+ }
+ } else {
+ String d = entry.getKey();
+ addConformanceEntry(curMethod, p, d, MediaBitstreams.K_UNSUPPORTED);
+ }
}
- if (size > limit || i + 1 == supportedBitstreams.size()) {
+ if (size > limit || i + 1 == bitstreams.size()) {
ReportProcessor processor;
- processor = new ProcessBitstreamsValidation(bitstreams, currentMethod);
+ processor = new ProcessBitstreamsValidation(toPush, curMethod);
processor.processDeviceReport(
device,
- currentMethod,
+ curMethod,
MediaBitstreams.KEY_BITSTREAMS_VALIDATION_TXT);
- bitstreams.clear();
+ toPush.clear();
size = 0;
n++;
}
diff --git a/hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/SupportedBitstreamsProcessor.java b/hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/SupportedBitstreamsProcessor.java
index 8a6c74e..87ccda4 100644
--- a/hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/SupportedBitstreamsProcessor.java
+++ b/hostsidetests/media/bitstreams/src/android/media/cts/bitstreams/SupportedBitstreamsProcessor.java
@@ -18,6 +18,8 @@
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
@@ -32,13 +34,14 @@
private final String mPrefix;
private final boolean mDebugTargetDevice;
private final Set<String> mSupportedBitstreams = new LinkedHashSet<>();
+ private final Map<String, Map<String, Boolean>> mDecodersForPath = new HashMap<>();
public SupportedBitstreamsProcessor() {
this("",false);
}
/**
- * @param prefix only bitstreams whose relative path starts with {@code prefix}
+ * @param prefix only bitstreams whose relative paths start with {@code prefix}
* would be processed
* @param debugTargetDevice whether to pause {@code device} for debugging
*/
@@ -48,12 +51,26 @@
}
/**
- * @return paths of supported devices on device
+ * @return paths to bitstreams that are supported on device
*/
public Set<String> getSupportedBitstreams() {
return mSupportedBitstreams;
}
+ /**
+ * @return paths to all bitstreams whose relative paths start with <code>prefix</code>
+ */
+ public Set<String> getBitstreams() {
+ return mDecodersForPath.keySet();
+ }
+
+ public Map<String, Boolean> getDecoderCapabilitiesForPath(String path) {
+ if (mDecodersForPath.containsKey(path)) {
+ return mDecodersForPath.get(path);
+ }
+ return Collections.emptyMap();
+ }
+
@Override
Map<String, String> getArgs() {
Map<String, String> args = new HashMap<>();
@@ -65,11 +82,29 @@
@Override
void process(ITestDevice device, String reportPath) throws DeviceNotAvailableException {
mSupportedBitstreams.clear();
- for (String path: getReportLines(device, reportPath)) {
- if (path.isEmpty()) {
- continue;
+ String[] lines = getReportLines(device, reportPath);
+ try {
+ for (int i = 0; i < lines.length;) {
+ String path = lines[i++];
+ int n = Integer.parseInt(lines[i++]);
+ for (int j = 0; j < n; j++) {
+ String name = lines[i++];
+ String status = lines[i++];
+ boolean supported = status.equals("true");
+ if (supported) {
+ mSupportedBitstreams.add(path);
+ }
+ Map<String, Boolean> decoderCapabilities;
+ if (mDecodersForPath.containsKey(path)) {
+ decoderCapabilities = mDecodersForPath.get(path);
+ } else {
+ mDecodersForPath.put(path, decoderCapabilities = new HashMap<>());
+ }
+ decoderCapabilities.put(name, supported);
+ }
}
- mSupportedBitstreams.add(path);
+ } catch (Exception e) {
+ CLog.w(e);
}
}
