Add histogram metrics for video playback judder
Bug: 234833109
Test: atest VideoRenderQualityTracker_test
Change-Id: I3a7f36384ab5e79f5e0e706a8f62a1e4854587a0
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 939f6d6..bd56b18 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -215,6 +215,11 @@
static const char *kCodecFreezeDistanceHistogram =
"android.media.mediacodec.freeze.distance.histogram";
+static const char *kCodecJudderCount = "android.media.mediacodec.judder.count";
+static const char *kCodecJudderScoreAverage = "android.media.mediacodec.judder.average";
+static const char *kCodecJudderScoreMax = "android.media.mediacodec.judder.max";
+static const char *kCodecJudderScoreHistogram = "android.media.mediacodec.judder.histogram";
+
/* -1: shaper disabled
>=0: number of fields changed */
static const char *kCodecShapingEnhanced = "android.media.mediacodec.shaped";
@@ -1135,6 +1140,13 @@
mediametrics_setInt64(mMetricsHandle, kCodecFreezeDistanceAverage, histogram.getAvg());
mediametrics_setString(mMetricsHandle, kCodecFreezeDistanceHistogram, histogram.emit());
}
+ if (m.judderScoreHistogram.getCount() >= 1) {
+ const MediaHistogram &histogram = m.judderScoreHistogram;
+ mediametrics_setInt64(mMetricsHandle, kCodecJudderCount, histogram.getCount());
+ mediametrics_setInt64(mMetricsHandle, kCodecJudderScoreAverage, histogram.getAvg());
+ mediametrics_setInt64(mMetricsHandle, kCodecJudderScoreMax, histogram.getMax());
+ mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogram, histogram.emit());
+ }
}
if (mLatencyHist.getCount() != 0 ) {
diff --git a/media/libstagefright/VideoRenderQualityTracker.cpp b/media/libstagefright/VideoRenderQualityTracker.cpp
index 5f0e969..f995e25 100644
--- a/media/libstagefright/VideoRenderQualityTracker.cpp
+++ b/media/libstagefright/VideoRenderQualityTracker.cpp
@@ -55,6 +55,10 @@
freezeDurationMsHistogramBuckets = {1, 20, 40, 60, 80, 100, 120, 150, 175, 225, 300, 400, 500};
freezeDistanceMsHistogramBuckets = {0, 20, 100, 400, 1000, 2000, 3000, 4000, 8000, 15000, 30000,
60000};
+
+ // Judder configuration
+ judderErrorToleranceUs = 2000;
+ judderScoreHistogramBuckets = {1, 4, 5, 9, 11, 20, 30, 40, 50, 60, 70, 80};
}
VideoRenderQualityTracker::VideoRenderQualityTracker() : mConfiguration(Configuration()) {
@@ -239,6 +243,14 @@
processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mMetrics);
mLastFreezeEndTimeUs = actualRenderTimeUs;
}
+
+ // Judder is computed on the prior video frame, not the current video frame
+ int64_t judderScore = computePreviousJudderScore(mActualFrameDurationUs,
+ mContentFrameDurationUs,
+ mConfiguration);
+ if (judderScore != 0) {
+ mMetrics.judderScoreHistogram.insert(judderScore);
+ }
}
void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
@@ -252,10 +264,53 @@
}
}
+int64_t VideoRenderQualityTracker::computePreviousJudderScore(
+ const FrameDurationUs &actualFrameDurationUs,
+ const FrameDurationUs &contentFrameDurationUs,
+ const Configuration &c) {
+ // If the frame before or after was dropped, then don't generate a judder score, since any
+ // problems with frame drops are scored as a freeze instead.
+ if (actualFrameDurationUs[0] == -1 || actualFrameDurationUs[1] == -1 ||
+ actualFrameDurationUs[2] == -1) {
+ return 0;
+ }
+
+ // Don't score judder for when playback is paused or rebuffering (long frame duration), or if
+ // the player is intentionally playing each frame at a slow rate (e.g. half-rate). If the long
+ // frame duration was unintentional, it is assumed that this will be coupled with a later frame
+ // drop, and be scored as a freeze instead of judder.
+ if (actualFrameDurationUs[1] >= 2 * contentFrameDurationUs[1]) {
+ return 0;
+ }
+
+ // The judder score is based on the error of this frame
+ int64_t errorUs = actualFrameDurationUs[1] - contentFrameDurationUs[1];
+ // Don't score judder if the previous frame has high error, but this frame has low error
+ if (abs(errorUs) < c.judderErrorToleranceUs) {
+ return 0;
+ }
+
+ // Add a penalty if this frame has judder that amplifies the problem introduced by previous
+ // judder, instead of catching up for the previous judder (50, 16, 16, 50) vs (50, 16, 50, 16)
+ int64_t previousErrorUs = actualFrameDurationUs[2] - contentFrameDurationUs[2];
+ // Don't add the pentalty for errors from the previous frame if the previous frame has low error
+ if (abs(previousErrorUs) >= c.judderErrorToleranceUs) {
+ errorUs = abs(errorUs) + abs(errorUs + previousErrorUs);
+ }
+
+ // Avoid scoring judder for 3:2 pulldown or other minimally-small frame duration errors
+ if (abs(errorUs) < contentFrameDurationUs[1] / 4) {
+ return 0;
+ }
+
+ return abs(errorUs) / 1000; // error in millis to keep numbers small
+}
+
void VideoRenderQualityTracker::configureHistograms(VideoRenderQualityMetrics &m,
const Configuration &c) {
m.freezeDurationMsHistogram.setup(c.freezeDurationMsHistogramBuckets);
m.freezeDistanceMsHistogram.setup(c.freezeDistanceMsHistogramBuckets);
+ m.judderScoreHistogram.setup(c.judderScoreHistogramBuckets);
}
int64_t VideoRenderQualityTracker::nowUs() {
diff --git a/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h b/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
index 45c822a..25ccfdb 100644
--- a/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
+++ b/media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h
@@ -63,6 +63,9 @@
// A histogram of the durations between each freeze.
MediaHistogram freezeDistanceMsHistogram;
+
+ // A histogram of the judder scores.
+ MediaHistogram judderScoreHistogram;
};
///////////////////////////////////////////////////////
@@ -113,6 +116,9 @@
// Freeze configuration
std::vector<int64_t> freezeDurationMsHistogramBuckets;
std::vector<int64_t> freezeDistanceMsHistogramBuckets;
+
+ int32_t judderErrorToleranceUs;
+ std::vector<int64_t> judderScoreHistogramBuckets;
};
VideoRenderQualityTracker();
@@ -197,6 +203,11 @@
static void processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
int64_t lastFreezeEndTimeUs, VideoRenderQualityMetrics &m);
+ // Compute a judder score for the previously-rendered frame.
+ static int64_t computePreviousJudderScore(const FrameDurationUs &actualRenderDurationUs,
+ const FrameDurationUs &contentRenderDurationUs,
+ const Configuration &c);
+
// Check to see if a discontinuity has occurred by examining the content time and the
// app-desired render time. If so, reset some internal state.
bool resetIfDiscontinuity(int64_t contentTimeUs, int64_t desiredRenderTimeUs);
diff --git a/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp b/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
index 2249a8b..efe1990 100644
--- a/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
+++ b/media/libstagefright/tests/VideoRenderQualityTracker_test.cpp
@@ -50,12 +50,13 @@
}
}
- void render(int numFrames) {
+ void render(int numFrames, float durationMs = -1) {
+ int64_t durationUs = durationMs < 0 ? mContentFrameDurationUs : durationMs * 1000;
for (int i = 0; i < numFrames; ++i) {
mVideoRenderQualityTracker.onFrameReleased(mMediaTimeUs);
mVideoRenderQualityTracker.onFrameRendered(mMediaTimeUs, mClockTimeNs);
mMediaTimeUs += mContentFrameDurationUs;
- mClockTimeNs += mContentFrameDurationUs * 1000;
+ mClockTimeNs += durationUs * 1000;
}
}
@@ -291,4 +292,207 @@
6 * 17) / 6);
}
+TEST_F(VideoRenderQualityTrackerTest, when60hz_hasNoJudder) {
+ Configuration c;
+ Helper h(16.66, c); // ~24Hz
+ h.render({16.66, 16.66, 16.66, 16.66, 16.66, 16.66, 16.66});
+ EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, whenSmallVariance60hz_hasNoJudder) {
+ Configuration c;
+ Helper h(16.66, c); // ~24Hz
+ h.render({14, 18, 14, 18, 14, 18, 14, 18});
+ EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, whenBadSmallVariance60Hz_hasJudder) {
+ Configuration c;
+ Helper h(16.66, c); // ~24Hz
+ h.render({14, 18, 14, /* no 18 between 14s */ 14, 18, 14, 18});
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, when30Hz_hasNoJudder) {
+ Configuration c;
+ Helper h(33.33, c);
+ h.render({33.33, 33.33, 33.33, 33.33, 33.33, 33.33});
+ EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, whenSmallVariance30Hz_hasNoJudder) {
+ Configuration c;
+ Helper h(33.33, c);
+ h.render({29.0, 35.0, 29.0, 35.0, 29.0, 35.0});
+ EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, whenBadSmallVariance30Hz_hasJudder) {
+ Configuration c;
+ Helper h(33.33, c);
+ h.render({29.0, 35.0, 29.0, /* no 35 between 29s */ 29.0, 35.0, 29.0, 35.0});
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, whenBad30HzTo60Hz_hasJudder) {
+ Configuration c;
+ Helper h(33.33, c);
+ h.render({33.33, 33.33, 50.0, /* frame stayed 1 vsync too long */ 16.66, 33.33, 33.33});
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 2); // note: 2 counts of judder
+}
+
+TEST_F(VideoRenderQualityTrackerTest, when24HzTo60Hz_hasNoJudder) {
+ Configuration c;
+ Helper h(41.66, c);
+ h.render({50.0, 33.33, 50.0, 33.33, 50.0, 33.33});
+ EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, when25HzTo60Hz_hasJudder) {
+ Configuration c;
+ Helper h(40, c);
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, when50HzTo60Hz_hasJudder) {
+ Configuration c;
+ Helper h(20, c);
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, when30HzTo50Hz_hasJudder) {
+ Configuration c;
+ Helper h(33.33, c);
+ h.render({40.0, 40.0, 40.0, 60.0});
+ h.render({40.0, 40.0, 40.0, 60.0});
+ h.render({40.0, 40.0, 40.0, 60.0});
+ h.render({40.0, 40.0, 40.0, 60.0});
+ h.render({40.0, 40.0, 40.0, 60.0});
+ EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, whenSmallVariancePulldown24HzTo60Hz_hasNoJudder) {
+ Configuration c;
+ Helper h(41.66, c);
+ h.render({52.0, 31.33, 52.0, 31.33, 52.0, 31.33});
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, whenBad24HzTo60Hz_hasJudder) {
+ Configuration c;
+ Helper h(41.66, c);
+ h.render({50.0, 33.33, 50.0, 33.33, /* no 50 between 33s */ 33.33, 50.0, 33.33});
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, capturesJudderScoreHistogram) {
+ Configuration c;
+ c.judderErrorToleranceUs = 2000;
+ c.judderScoreHistogramBuckets = {1, 5, 8};
+ Helper h(16, c);
+ h.render({16, 16, 23, 16, 16, 10, 16, 4, 16, 20, 16, 16});
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.emit(), "0{1,2}1");
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 4);
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getMin(), 4);
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getMax(), 12);
+ EXPECT_EQ(h.getMetrics().judderScoreHistogram.getAvg(), (7 + 6 + 12 + 4) / 4);
+}
+
+TEST_F(VideoRenderQualityTrackerTest, ranksJudderScoresInOrder) {
+ // Each rendering is ranked from best to worst from a user experience
+ Configuration c;
+ c.judderErrorToleranceUs = 2000;
+ c.judderScoreHistogramBuckets = {0, 1000};
+ int64_t previousScore = 0;
+
+ // 30fps poorly displayed at 60Hz
+ {
+ Helper h(33.33, c);
+ h.render({33.33, 33.33, 16.66, 50.0, 33.33, 33.33});
+ int64_t scoreBad30fpsTo60Hz = h.getMetrics().judderScoreHistogram.getMax();
+ EXPECT_GT(scoreBad30fpsTo60Hz, previousScore);
+ previousScore = scoreBad30fpsTo60Hz;
+ }
+
+ // 25fps displayed at 60hz
+ {
+ Helper h(40, c);
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ h.render({33.33, 33.33, 50.0});
+ int64_t score25fpsTo60hz = h.getMetrics().judderScoreHistogram.getMax();
+ EXPECT_GT(score25fpsTo60hz, previousScore);
+ previousScore = score25fpsTo60hz;
+ }
+
+ // 50fps displayed at 60hz
+ {
+ Helper h(20, c);
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ h.render({16.66, 16.66, 16.66, 33.33});
+ int64_t score50fpsTo60hz = h.getMetrics().judderScoreHistogram.getMax();
+ EXPECT_GT(score50fpsTo60hz, previousScore);
+ previousScore = score50fpsTo60hz;
+ }
+
+ // 24fps poorly displayed at 60Hz
+ {
+ Helper h(41.66, c);
+ h.render({50.0, 33.33, 50.0, 33.33, 33.33, 50.0, 33.33});
+ int64_t scoreBad24HzTo60Hz = h.getMetrics().judderScoreHistogram.getMax();
+ EXPECT_GT(scoreBad24HzTo60Hz, previousScore);
+ previousScore = scoreBad24HzTo60Hz;
+ }
+
+ // 30fps displayed at 50hz
+ {
+ Helper h(33.33, c);
+ h.render({40.0, 40.0, 40.0, 60.0});
+ h.render({40.0, 40.0, 40.0, 60.0});
+ h.render({40.0, 40.0, 40.0, 60.0});
+ h.render({40.0, 40.0, 40.0, 60.0});
+ h.render({40.0, 40.0, 40.0, 60.0});
+ int64_t score30fpsTo50hz = h.getMetrics().judderScoreHistogram.getMax();
+ EXPECT_GT(score30fpsTo50hz, previousScore);
+ previousScore = score30fpsTo50hz;
+ }
+
+ // 24fps displayed at 50Hz
+ {
+ Helper h(41.66, c);
+ h.render(40.0, 11);
+ h.render(60.0, 1);
+ h.render(40.0, 11);
+ h.render(60.0, 1);
+ h.render(40.0, 11);
+ int64_t score24HzTo50Hz = h.getMetrics().judderScoreHistogram.getMax();
+ EXPECT_GT(score24HzTo50Hz, previousScore);
+ previousScore = score24HzTo50Hz;
+ }
+}
+
} // android
