Merge "mediapc: evaluator changes for frame drop tests" am: 5ace1fedc5 am: b8870ad2ba am: 0aa4c98dad am: efb20268aa am: b946f7b506
Original change: https://android-review.googlesource.com/c/platform/cts/+/2087526
Change-Id: I956dab11a2c4f1797a7dfd7098e9ef306dfe5944
Merged-In: I956dab11a2c4f1797a7dfd7098e9ef306dfe5944
Merged-In: I7f74919207b302c8de0af4a46c0dfcad62afdb55
Merged-In: I20df01a0de9d1dcdec3b80c486b22b42eb8d5fb6
Merged-In: I1a7cfd9f884525528d02d38a21bb1b1b6dfce981
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
(cherry picked from commit 76822de5a92cf46081384788dfa9fe4b62e697b6)
diff --git a/tests/mediapc/common/src/android/mediapc/cts/common/PerformanceClassEvaluator.java b/tests/mediapc/common/src/android/mediapc/cts/common/PerformanceClassEvaluator.java
index 70340af..71bb8c8 100644
--- a/tests/mediapc/common/src/android/mediapc/cts/common/PerformanceClassEvaluator.java
+++ b/tests/mediapc/common/src/android/mediapc/cts/common/PerformanceClassEvaluator.java
@@ -22,18 +22,12 @@
import android.os.Build;
-import androidx.test.filters.SmallTest;
-
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
import org.junit.rules.TestName;
-import org.junit.Test;
import java.util.HashSet;
import java.util.Set;
-import java.util.HashMap;
-import java.util.Map;
/**
* Logs a set of measurements and results for defined performance class requirements.
@@ -285,6 +279,95 @@
}
}
+ // used for requirements [2.2.7.1/5.3/H-1-1], [2.2.7.1/5.3/H-1-2]
+ public static class FrameDropRequirement extends Requirement {
+ private static final String TAG = FrameDropRequirement.class.getSimpleName();
+
+ private FrameDropRequirement(String id, RequiredMeasurement<?>... reqs) {
+ super(id, reqs);
+ }
+
+ public void setFramesDropped(int framesDropped) {
+ this.setMeasuredValue(RequirementConstants.FRAMES_DROPPED, framesDropped);
+ }
+
+ /**
+ * [2.2.7.1/5.3/H-1-1] MUST NOT drop more than 1 frames in 10 seconds (i.e less than 0.333
+ * percent frame drop) for a 1080p 30 fps video session under load. Load is defined as a
+ * concurrent 1080p to 720p video-only transcoding session using hardware video codecs,
+ * as well as a 128 kbps AAC audio playback.
+ */
+ public static FrameDropRequirement createR5_3__H_1_1_R() {
+ RequiredMeasurement<Integer> frameDropped = RequiredMeasurement
+ .<Integer>builder()
+ .setId(RequirementConstants.FRAMES_DROPPED)
+ .setPredicate(RequirementConstants.INTEGER_LTE)
+ // MUST NOT drop more than 1 frame in 10 seconds so 3 frames for 30 seconds
+ .addRequiredValue(Build.VERSION_CODES.R, 3)
+ .build();
+
+ return new FrameDropRequirement(RequirementConstants.R5_3__H_1_1, frameDropped);
+ }
+
+ /**
+ * [2.2.7.1/5.3/H-1-2] MUST NOT drop more than 1 frame in 10 seconds during a video
+ * resolution change in a 30 fps video session under load. Load is defined as a
+ * concurrent 1080p to 720p video-only transcoding session using hardware video codecs,
+ * as well as a 128Kbps AAC audio playback.
+ */
+ public static FrameDropRequirement createR5_3__H_1_2_R() {
+ RequiredMeasurement<Integer> frameDropped = RequiredMeasurement
+ .<Integer>builder()
+ .setId(RequirementConstants.FRAMES_DROPPED)
+ .setPredicate(RequirementConstants.INTEGER_LTE)
+ // MUST NOT drop more than 1 frame in 10 seconds so 3 frames for 30 seconds
+ .addRequiredValue(Build.VERSION_CODES.R, 3)
+ .build();
+
+ return new FrameDropRequirement(RequirementConstants.R5_3__H_1_2, frameDropped);
+ }
+
+ /**
+ * [2.2.7.1/5.3/H-1-1] MUST NOT drop more than 2(S) / 1(T) frames in 10 seconds for a
+ * 1080p 60 fps video session under load. Load is defined as a concurrent 1080p to 720p
+ * video-only transcoding session using hardware video codecs, as well as a 128 kbps AAC
+ * audio playback.
+ */
+ public static FrameDropRequirement createR5_3__H_1_1_ST() {
+ RequiredMeasurement<Integer> frameDropped = RequiredMeasurement
+ .<Integer>builder()
+ .setId(RequirementConstants.FRAMES_DROPPED)
+ .setPredicate(RequirementConstants.INTEGER_LTE)
+ // MUST NOT drop more than 2 frame in 10 seconds so 6 frames for 30 seconds
+ .addRequiredValue(Build.VERSION_CODES.S, 6)
+ // MUST NOT drop more than 1 frame in 10 seconds so 3 frames for 30 seconds
+ .addRequiredValue(Build.VERSION_CODES.TIRAMISU, 3)
+ .build();
+
+ return new FrameDropRequirement(RequirementConstants.R5_3__H_1_1, frameDropped);
+ }
+
+ /**
+ * [2.2.7.1/5.3/H-1-2] MUST NOT drop more than 2(S) / 1(T) frames in 10 seconds during a
+ * video resolution change in a 60 fps video session under load. Load is defined as a
+ * concurrent 1080p to 720p video-only transcoding session using hardware video codecs,
+ * as well as a 128Kbps AAC audio playback.
+ */
+ public static FrameDropRequirement createR5_3__H_1_2_ST() {
+ RequiredMeasurement<Integer> frameDropped = RequiredMeasurement
+ .<Integer>builder()
+ .setId(RequirementConstants.FRAMES_DROPPED)
+ .setPredicate(RequirementConstants.INTEGER_LTE)
+ // MUST NOT drop more than 2 frame in 10 seconds so 6 frames for 30 seconds
+ .addRequiredValue(Build.VERSION_CODES.S, 6)
+ // MUST NOT drop more than 1 frame in 10 seconds so 3 frames for 30 seconds
+ .addRequiredValue(Build.VERSION_CODES.TIRAMISU, 3)
+ .build();
+
+ return new FrameDropRequirement(RequirementConstants.R5_3__H_1_2, frameDropped);
+ }
+ }
+
private <R extends Requirement> R addRequirement(R req) {
if (!this.mRequirements.add(req)) {
throw new IllegalStateException("Requirement " + req.id() + " already added");
@@ -318,6 +401,22 @@
return this.<MemoryRequirement>addRequirement(MemoryRequirement.createR7_6_1__H_2_1());
}
+ public FrameDropRequirement addR5_3__H_1_1_R() {
+ return this.addRequirement(FrameDropRequirement.createR5_3__H_1_1_R());
+ }
+
+ public FrameDropRequirement addR5_3__H_1_2_R() {
+ return this.addRequirement(FrameDropRequirement.createR5_3__H_1_2_R());
+ }
+
+ public FrameDropRequirement addR5_3__H_1_1_ST() {
+ return this.addRequirement(FrameDropRequirement.createR5_3__H_1_1_ST());
+ }
+
+ public FrameDropRequirement addR5_3__H_1_2_ST() {
+ return this.addRequirement(FrameDropRequirement.createR5_3__H_1_2_ST());
+ }
+
public CodecInitLatencyRequirement addR5_1__H_1_7() {
return this.addRequirement(CodecInitLatencyRequirement.createR5_1__H_1_7());
}
diff --git a/tests/mediapc/src/android/mediapc/cts/AdaptivePlaybackFrameDropTest.java b/tests/mediapc/src/android/mediapc/cts/AdaptivePlaybackFrameDropTest.java
index 49ae4a1..9d857d2 100644
--- a/tests/mediapc/src/android/mediapc/cts/AdaptivePlaybackFrameDropTest.java
+++ b/tests/mediapc/src/android/mediapc/cts/AdaptivePlaybackFrameDropTest.java
@@ -19,19 +19,17 @@
import static org.junit.Assert.assertTrue;
import android.media.MediaCodecInfo;
+import android.mediapc.cts.common.PerformanceClassEvaluator;
import android.mediapc.cts.common.Utils;
-import android.os.Build;
import androidx.test.filters.LargeTest;
-import androidx.test.platform.app.InstrumentationRegistry;
import com.android.compatibility.common.util.CddTest;
-import com.android.compatibility.common.util.DeviceReportLog;
-import com.android.compatibility.common.util.ResultType;
-import com.android.compatibility.common.util.ResultUnit;
import org.junit.Assume;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -49,6 +47,9 @@
super(mimeType, decoderName, isAsync);
}
+ @Rule
+ public final TestName mTestName = new TestName();
+
// Returns the list of parameters with mimeTypes and their hardware decoders supporting the
// AdaptivePlayback feature combining with sync and async modes.
// Parameters {0}_{1}_{2} -- Mime_DecoderName_isAsync
@@ -58,29 +59,14 @@
MediaCodecInfo.CodecCapabilities.FEATURE_AdaptivePlayback});
}
- private void testAdaptivePlaybackFrameDrop(int frameRate) throws Exception {
+ private int testAdaptivePlaybackFrameDrop(int frameRate) throws Exception {
String[] testFiles = frameRate == 30 ?
new String[]{m1080p30FpsTestFiles.get(mMime), m540p30FpsTestFiles.get(mMime)} :
new String[]{m1080p60FpsTestFiles.get(mMime), m540p60FpsTestFiles.get(mMime)};
PlaybackFrameDrop playbackFrameDrop = new PlaybackFrameDrop(mMime, mDecoderName, testFiles,
mSurface, frameRate, mIsAsync);
- int frameDropCount = playbackFrameDrop.getFrameDropCount();
- if (Utils.isPerfClass()) {
- assertTrue("Adaptive Playback FrameDrop count for mime: " + mMime + ", decoder: "
- + mDecoderName + ", FrameRate: " + frameRate
- + ", is not as expected. act/exp: " + frameDropCount + "/"
- + MAX_FRAME_DROP_FOR_30S, frameDropCount <= MAX_FRAME_DROP_FOR_30S);
- } else {
- int pc = getAchievedPerfClass(frameRate, frameDropCount);
- DeviceReportLog log = new DeviceReportLog("MediaPerformanceClassLogs",
- "AdaptiveFrameDrop_" + mDecoderName);
- log.addValue("decoder", mDecoderName, ResultType.NEUTRAL, ResultUnit.NONE);
- log.addValue("adaptive_frame_drops_for_30sec", frameDropCount, ResultType.LOWER_BETTER,
- ResultUnit.NONE);
- log.setSummary("CDD 2.2.7.1/5.3/H-1-2 performance_class", pc, ResultType.NEUTRAL,
- ResultUnit.NONE);
- log.submit(InstrumentationRegistry.getInstrumentation());
- }
+
+ return playbackFrameDrop.getFrameDropCount();
}
/**
@@ -94,9 +80,16 @@
@CddTest(requirement = "2.2.7.1/5.3/H-1-2")
public void test30Fps() throws Exception {
Assume.assumeTrue("Test is limited to R performance class devices or devices that do not " +
- "advertise performance class",
- Utils.isRPerfClass() || !Utils.isPerfClass());
- testAdaptivePlaybackFrameDrop(30);
+ "advertise performance class",
+ Utils.isRPerfClass() || !Utils.isPerfClass());
+
+ PerformanceClassEvaluator pce = new PerformanceClassEvaluator(this.mTestName);
+ PerformanceClassEvaluator.FrameDropRequirement r5_3__H_1_2_R = pce.addR5_3__H_1_2_R();
+
+ int framesDropped = testAdaptivePlaybackFrameDrop(30);
+
+ r5_3__H_1_2_R.setFramesDropped(framesDropped);
+ pce.submitAndCheck();
}
/**
@@ -110,8 +103,15 @@
@CddTest(requirement = "2.2.7.1/5.3/H-1-2")
public void test60Fps() throws Exception {
Assume.assumeTrue("Test is limited to S/T performance class devices or devices that do " +
- "not advertise performance class",
- Utils.isSPerfClass() || Utils.isTPerfClass() || !Utils.isPerfClass());
- testAdaptivePlaybackFrameDrop(60);
+ "not advertise performance class",
+ Utils.isSPerfClass() || Utils.isTPerfClass() || !Utils.isPerfClass());
+
+ PerformanceClassEvaluator pce = new PerformanceClassEvaluator(this.mTestName);
+ PerformanceClassEvaluator.FrameDropRequirement r5_3__H_1_2_ST = pce.addR5_3__H_1_2_ST();
+
+ int framesDropped = testAdaptivePlaybackFrameDrop(60);
+
+ r5_3__H_1_2_ST.setFramesDropped(framesDropped);
+ pce.submitAndCheck();
}
}
diff --git a/tests/mediapc/src/android/mediapc/cts/FrameDropTest.java b/tests/mediapc/src/android/mediapc/cts/FrameDropTest.java
index fd015aa..4e9409e 100644
--- a/tests/mediapc/src/android/mediapc/cts/FrameDropTest.java
+++ b/tests/mediapc/src/android/mediapc/cts/FrameDropTest.java
@@ -16,26 +16,18 @@
package android.mediapc.cts;
-import static org.junit.Assert.assertTrue;
-
+import android.mediapc.cts.common.PerformanceClassEvaluator;
import android.mediapc.cts.common.Utils;
-import android.os.Build;
-
import androidx.test.filters.LargeTest;
-import androidx.test.platform.app.InstrumentationRegistry;
-
import com.android.compatibility.common.util.CddTest;
-import com.android.compatibility.common.util.DeviceReportLog;
-import com.android.compatibility.common.util.ResultType;
-import com.android.compatibility.common.util.ResultUnit;
-
+import java.util.Collection;
import org.junit.Assume;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import java.util.Collection;
-
/**
* The following test class validates the frame drops of a playback for the hardware decoders
* under the load condition (Transcode + Audio Playback).
@@ -48,6 +40,9 @@
super(mimeType, decoderName, isAsync);
}
+ @Rule
+ public final TestName mTestName = new TestName();
+
// Returns the list of parameters with mimeTypes and their hardware decoders
// combining with sync and async modes.
// Parameters {0}_{1}_{2} -- Mime_DecoderName_isAsync
@@ -56,29 +51,13 @@
return prepareArgumentsList(null);
}
- private void testDecodeToSurface(int frameRate) throws Exception {
+ private int testDecodeToSurface(int frameRate) throws Exception {
String[] testFiles = frameRate == 30 ?
new String[]{m1080p30FpsTestFiles.get(mMime)} :
new String[]{m1080p60FpsTestFiles.get(mMime)};
PlaybackFrameDrop playbackFrameDrop = new PlaybackFrameDrop(mMime, mDecoderName, testFiles,
mSurface, frameRate, mIsAsync);
- int frameDropCount = playbackFrameDrop.getFrameDropCount();
- if (Utils.isPerfClass()) {
- assertTrue("FrameDrop count for mime: " + mMime + ", decoder: " + mDecoderName
- + ", FrameRate: " + frameRate + ", is not as expected. act/exp: "
- + frameDropCount + "/" + MAX_FRAME_DROP_FOR_30S,
- frameDropCount <= MAX_FRAME_DROP_FOR_30S);
- } else {
- int pc = getAchievedPerfClass(frameRate, frameDropCount);
- DeviceReportLog log = new DeviceReportLog("MediaPerformanceClassLogs",
- "FrameDrop_" + mDecoderName);
- log.addValue("decoder", mDecoderName, ResultType.NEUTRAL, ResultUnit.NONE);
- log.addValue("frame_drops_for_30sec_" + frameRate + "fps", frameDropCount,
- ResultType.LOWER_BETTER, ResultUnit.NONE);
- log.setSummary("CDD 2.2.7.1/5.3/H-1-1 performance_class", pc, ResultType.NEUTRAL,
- ResultUnit.NONE);
- log.submit(InstrumentationRegistry.getInstrumentation());
- }
+ return playbackFrameDrop.getFrameDropCount();
}
/**
@@ -93,7 +72,13 @@
Assume.assumeTrue("Test is limited to R performance class devices or devices that do not " +
"advertise performance class",
Utils.isRPerfClass() || !Utils.isPerfClass());
- testDecodeToSurface(30);
+
+ PerformanceClassEvaluator pce = new PerformanceClassEvaluator(this.mTestName);
+ PerformanceClassEvaluator.FrameDropRequirement r5_3__H_1_1_R = pce.addR5_3__H_1_1_R();
+
+ int framesDropped = testDecodeToSurface(30);
+ r5_3__H_1_1_R.setFramesDropped(framesDropped);
+ pce.submitAndCheck();
}
/**
@@ -108,6 +93,12 @@
Assume.assumeTrue("Test is limited to S/T performance class devices or devices that do " +
"not advertise performance class",
Utils.isSPerfClass() || Utils.isTPerfClass() || !Utils.isPerfClass());
- testDecodeToSurface(60);
+
+ PerformanceClassEvaluator pce = new PerformanceClassEvaluator(this.mTestName);
+ PerformanceClassEvaluator.FrameDropRequirement r5_3__H_1_1_ST = pce.addR5_3__H_1_1_ST();
+
+ int framesDropped = testDecodeToSurface(60);
+ r5_3__H_1_1_ST.setFramesDropped(framesDropped);
+ pce.submitAndCheck();
}
}
