NuPlayer: adjust temporal-layers to achieve smooth play
For multi-layer-encoded content (with hierarchical referencing),
select a subset of layers based on playback speed.
Bug: 27596987
Change-Id: I824c699f3c2b80bb516c1b44493eda72239291ac
(cherry picked from commit 64ad0e0b2b5f992052d287e82ebd79f9f892dcbf)
diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.cpp b/media/libmediaplayerservice/nuplayer/GenericSource.cpp
index 3d836c8..1a0539d 100644
--- a/media/libmediaplayerservice/nuplayer/GenericSource.cpp
+++ b/media/libmediaplayerservice/nuplayer/GenericSource.cpp
@@ -1298,6 +1298,13 @@
}
#endif
+ if (trackType == MEDIA_TRACK_TYPE_VIDEO) {
+ int32_t layerId;
+ if (mb->meta_data()->findInt32(kKeyTemporalLayerId, &layerId)) {
+ meta->setInt32("temporal-layer-id", layerId);
+ }
+ }
+
if (trackType == MEDIA_TRACK_TYPE_TIMEDTEXT) {
const char *mime;
CHECK(mTimedTextTrack.mSource != NULL
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index 4504b58..134da14 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -784,12 +784,9 @@
}
if (mVideoDecoder != NULL) {
- float rate = getFrameRate();
- if (rate > 0) {
- sp<AMessage> params = new AMessage();
- params->setFloat("operating-rate", rate * mPlaybackSettings.mSpeed);
- mVideoDecoder->setParameters(params);
- }
+ sp<AMessage> params = new AMessage();
+ params->setFloat("playback-speed", mPlaybackSettings.mSpeed);
+ mVideoDecoder->setParameters(params);
}
sp<AMessage> response = new AMessage;
@@ -1680,6 +1677,27 @@
return err;
}
}
+
+ if (!audio) {
+ sp<AMessage> params = new AMessage();
+ float rate = getFrameRate();
+ if (rate > 0) {
+ params->setFloat("frame-rate-total", rate);
+ }
+
+ sp<MetaData> fileMeta = getFileMeta();
+ if (fileMeta != NULL) {
+ int32_t videoTemporalLayerCount;
+ if (fileMeta->findInt32(kKeyTemporalLayerCount, &videoTemporalLayerCount)
+ && videoTemporalLayerCount > 0) {
+ params->setInt32("temporal-layer-count", videoTemporalLayerCount);
+ }
+ }
+
+ if (params->countEntries() > 0) {
+ (*decoder)->setParameters(params);
+ }
+ }
return OK;
}
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
index 4678956..fa19410 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
@@ -41,6 +41,12 @@
namespace android {
+static float kDisplayRefreshingRate = 60.f;
+
+// The default total video frame rate of a stream when that info is not available from
+// the source.
+static float kDefaultVideoFrameRateTotal = 30.f;
+
static inline bool getAudioDeepBufferSetting() {
return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
}
@@ -69,11 +75,17 @@
mIsSecure(false),
mFormatChangePending(false),
mTimeChangePending(false),
+ mFrameRateTotal(kDefaultVideoFrameRateTotal),
+ mPlaybackSpeed(1.0f),
+ mNumVideoTemporalLayerTotal(1),
+ mNumVideoTemporalLayerAllowed(1),
+ mCurrentMaxVideoTemporalLayerId(0),
mResumePending(false),
mComponentName("decoder") {
mCodecLooper = new ALooper;
mCodecLooper->setName("NPDecoder-CL");
mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
+ mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
}
NuPlayer::Decoder::~Decoder() {
@@ -329,11 +341,73 @@
}
void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) {
- if (mCodec == NULL) {
- ALOGW("onSetParameters called before codec is created.");
- return;
+ bool needAdjustLayers = false;
+ float frameRateTotal;
+ if (params->findFloat("frame-rate-total", &frameRateTotal)
+ && mFrameRateTotal != frameRateTotal) {
+ needAdjustLayers = true;
+ mFrameRateTotal = frameRateTotal;
}
- mCodec->setParameters(params);
+
+ int32_t numVideoTemporalLayerTotal;
+ if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
+ && numVideoTemporalLayerTotal > 0
+ && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
+ && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
+ needAdjustLayers = true;
+ mNumVideoTemporalLayerTotal = numVideoTemporalLayerTotal;
+ }
+
+ if (needAdjustLayers) {
+ // TODO: For now, layer fps is calculated for some specific architectures.
+ // But it really should be extracted from the stream.
+ mVideoTemporalLayerAggregateFps[0] =
+ mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
+ for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
+ mVideoTemporalLayerAggregateFps[i] =
+ mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
+ + mVideoTemporalLayerAggregateFps[i - 1];
+ }
+ }
+
+ float playbackSpeed;
+ if (params->findFloat("playback-speed", &playbackSpeed)
+ && mPlaybackSpeed != playbackSpeed) {
+ needAdjustLayers = true;
+ mPlaybackSpeed = playbackSpeed;
+ }
+
+ if (needAdjustLayers) {
+ int32_t layerId;
+ for (layerId = 0; layerId < mNumVideoTemporalLayerTotal; ++layerId) {
+ if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
+ > kDisplayRefreshingRate) {
+ --layerId;
+ break;
+ }
+ }
+ if (layerId < 0) {
+ layerId = 0;
+ } else if (layerId >= mNumVideoTemporalLayerTotal) {
+ layerId = mNumVideoTemporalLayerTotal - 1;
+ }
+ mNumVideoTemporalLayerAllowed = layerId + 1;
+ if (mCurrentMaxVideoTemporalLayerId > layerId) {
+ mCurrentMaxVideoTemporalLayerId = layerId;
+ }
+ ALOGV("onSetParameters: allowed layers=%d, current max layerId=%d",
+ mNumVideoTemporalLayerAllowed, mCurrentMaxVideoTemporalLayerId);
+
+ if (mCodec == NULL) {
+ ALOGW("onSetParameters called before codec is created.");
+ return;
+ }
+
+ sp<AMessage> codecParams = new AMessage();
+ codecParams->setFloat("operating-rate",
+ mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed);
+ mCodec->setParameters(codecParams);
+ }
}
void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
@@ -742,13 +816,27 @@
}
dropAccessUnit = false;
- if (!mIsAudio
- && !mIsSecure
- && mRenderer->getVideoLateByUs() > 100000ll
- && mIsVideoAVC
- && !IsAVCReferenceFrame(accessUnit)) {
- dropAccessUnit = true;
- ++mNumInputFramesDropped;
+ if (!mIsAudio && !mIsSecure) {
+ int32_t layerId = 0;
+ if (mRenderer->getVideoLateByUs() > 100000ll
+ && mIsVideoAVC
+ && !IsAVCReferenceFrame(accessUnit)) {
+ dropAccessUnit = true;
+ } else if (accessUnit->meta()->findInt32("temporal-layer-id", &layerId)) {
+ // Add only one layer each time.
+ if (layerId > mCurrentMaxVideoTemporalLayerId + 1
+ || layerId >= mNumVideoTemporalLayerAllowed) {
+ dropAccessUnit = true;
+ ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
+ layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
+ mCurrentMaxVideoTemporalLayerId);
+ } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
+ mCurrentMaxVideoTemporalLayerId = layerId;
+ }
+ }
+ if (dropAccessUnit) {
+ ++mNumInputFramesDropped;
+ }
}
} while (dropAccessUnit);
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
index ae08b4b..0c619ed 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
@@ -57,6 +57,10 @@
kWhatSetVideoSurface = 'sSur'
};
+ enum {
+ kMaxNumVideoTemporalLayers = 32,
+ };
+
sp<Surface> mSurface;
sp<Source> mSource;
@@ -90,6 +94,12 @@
bool mIsSecure;
bool mFormatChangePending;
bool mTimeChangePending;
+ float mFrameRateTotal;
+ float mPlaybackSpeed;
+ int32_t mNumVideoTemporalLayerTotal;
+ int32_t mNumVideoTemporalLayerAllowed;
+ int32_t mCurrentMaxVideoTemporalLayerId;
+ float mVideoTemporalLayerAggregateFps[kMaxNumVideoTemporalLayers];
bool mResumePending;
AString mComponentName;
