blob: 85027ced511e11f871494c4d0224c35292beb17f [file] [log] [blame]
/*
* Copyright 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//#define LOG_NDEBUG 0
#define LOG_TAG "MediaSync"
#include <inttypes.h>
#include <gui/BufferQueue.h>
#include <gui/IGraphicBufferConsumer.h>
#include <gui/IGraphicBufferProducer.h>
#include <media/AudioTrack.h>
#include <media/stagefright/MediaClock.h>
#include <media/stagefright/MediaSync.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/ALooper.h>
#include <media/stagefright/foundation/AMessage.h>
#include <ui/GraphicBuffer.h>
// Maximum late time allowed for a video frame to be rendered. When a video
// frame arrives later than this number, it will be discarded without rendering.
static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll;
namespace android {
// static
sp<MediaSync> MediaSync::create() {
sp<MediaSync> sync = new MediaSync();
sync->mLooper->registerHandler(sync);
return sync;
}
MediaSync::MediaSync()
: mIsAbandoned(false),
mMutex(),
mReleaseCondition(),
mNumOutstandingBuffers(0),
mNativeSampleRateInHz(0),
mNumFramesWritten(0),
mHasAudio(false),
mNextBufferItemMediaUs(-1),
mPlaybackRate(0.0) {
mMediaClock = new MediaClock;
// initialize settings
mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
mPlaybackSettings.mSpeed = mPlaybackRate;
mLooper = new ALooper;
mLooper->setName("MediaSync");
mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
}
MediaSync::~MediaSync() {
if (mInput != NULL) {
mInput->consumerDisconnect();
}
if (mOutput != NULL) {
mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
}
if (mLooper != NULL) {
mLooper->unregisterHandler(id());
mLooper->stop();
}
}
status_t MediaSync::configureSurface(const sp<IGraphicBufferProducer> &output) {
Mutex::Autolock lock(mMutex);
// TODO: support suface change.
if (mOutput != NULL) {
ALOGE("configureSurface: output surface has already been configured.");
return INVALID_OPERATION;
}
if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
ALOGE("configureSurface: output surface is used as sync source and cannot be removed.");
return INVALID_OPERATION;
}
if (output != NULL) {
IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
sp<OutputListener> listener(new OutputListener(this));
IInterface::asBinder(output)->linkToDeath(listener);
status_t status =
output->connect(listener,
NATIVE_WINDOW_API_MEDIA,
true /* producerControlledByApp */,
&queueBufferOutput);
if (status != NO_ERROR) {
ALOGE("configureSurface: failed to connect (%d)", status);
return status;
}
mOutput = output;
}
return NO_ERROR;
}
// |audioTrack| is used only for querying information.
status_t MediaSync::configureAudioTrack(const sp<AudioTrack> &audioTrack) {
Mutex::Autolock lock(mMutex);
// TODO: support audio track change.
if (mAudioTrack != NULL) {
ALOGE("configureAudioTrack: audioTrack has already been configured.");
return INVALID_OPERATION;
}
if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
ALOGE("configureAudioTrack: audioTrack is used as sync source and cannot be removed.");
return INVALID_OPERATION;
}
if (audioTrack != NULL) {
// check if audio track supports the playback settings
if (mPlaybackSettings.mSpeed != 0.f
&& audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
ALOGE("playback settings are not supported by the audio track");
return INVALID_OPERATION;
}
uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
if (nativeSampleRateInHz <= 0) {
ALOGE("configureAudioTrack: native sample rate should be positive.");
return BAD_VALUE;
}
mAudioTrack = audioTrack;
mNativeSampleRateInHz = nativeSampleRateInHz;
(void)setPlaybackSettings_l(mPlaybackSettings);
}
else {
mAudioTrack = NULL;
mNativeSampleRateInHz = 0;
}
// potentially resync to new source
resync_l();
return OK;
}
status_t MediaSync::createInputSurface(
sp<IGraphicBufferProducer> *outBufferProducer) {
if (outBufferProducer == NULL) {
return BAD_VALUE;
}
Mutex::Autolock lock(mMutex);
if (mOutput == NULL) {
return NO_INIT;
}
if (mInput != NULL) {
return INVALID_OPERATION;
}
sp<IGraphicBufferProducer> bufferProducer;
sp<IGraphicBufferConsumer> bufferConsumer;
BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer);
sp<InputListener> listener(new InputListener(this));
IInterface::asBinder(bufferConsumer)->linkToDeath(listener);
status_t status =
bufferConsumer->consumerConnect(listener, false /* controlledByApp */);
if (status == NO_ERROR) {
bufferConsumer->setConsumerName(String8("MediaSync"));
*outBufferProducer = bufferProducer;
mInput = bufferConsumer;
}
return status;
}
void MediaSync::resync_l() {
AVSyncSource src = mSyncSettings.mSource;
if (src == AVSYNC_SOURCE_DEFAULT) {
if (mAudioTrack != NULL) {
src = AVSYNC_SOURCE_AUDIO;
} else {
src = AVSYNC_SOURCE_SYSTEM_CLOCK;
}
}
// TODO: resync ourselves to the current clock (e.g. on sync source change)
updatePlaybackRate_l(mPlaybackRate);
}
void MediaSync::updatePlaybackRate_l(float rate) {
if (rate > mPlaybackRate) {
mNextBufferItemMediaUs = -1;
}
mPlaybackRate = rate;
mMediaClock->setPlaybackRate(rate);
onDrainVideo_l();
}
sp<const MediaClock> MediaSync::getMediaClock() {
return mMediaClock;
}
status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
Mutex::Autolock lock(mMutex);
// User should check the playback rate if it doesn't want to receive a
// huge number for play time.
if (mPlaybackRate == 0.0f) {
*outTimeUs = INT64_MAX;
return OK;
}
uint32_t numFramesPlayed = 0;
if (mAudioTrack != NULL) {
status_t res = mAudioTrack->getPosition(&numFramesPlayed);
if (res != OK) {
return res;
}
}
int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed;
if (numPendingFrames < 0) {
numPendingFrames = 0;
ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative.");
}
double timeUs = numPendingFrames * 1000000.0
/ (mNativeSampleRateInHz * (double)mPlaybackRate);
if (timeUs > (double)INT64_MAX) {
// Overflow.
*outTimeUs = INT64_MAX;
ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames "
"is too high, possibly due to super low playback rate(%f)", mPlaybackRate);
} else {
*outTimeUs = (int64_t)timeUs;
}
return OK;
}
status_t MediaSync::updateQueuedAudioData(
size_t sizeInBytes, int64_t presentationTimeUs) {
if (sizeInBytes == 0) {
return OK;
}
Mutex::Autolock lock(mMutex);
if (mAudioTrack == NULL) {
ALOGW("updateQueuedAudioData: audioTrack has NOT been configured.");
return INVALID_OPERATION;
}
int64_t numFrames = sizeInBytes / mAudioTrack->frameSize();
int64_t maxMediaTimeUs = presentationTimeUs
+ getDurationIfPlayedAtNativeSampleRate_l(numFrames);
int64_t nowUs = ALooper::GetNowUs();
int64_t nowMediaUs = presentationTimeUs
- getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten)
+ getPlayedOutAudioDurationMedia_l(nowUs);
mNumFramesWritten += numFrames;
int64_t oldRealTime = -1;
if (mNextBufferItemMediaUs != -1) {
oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
}
mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs);
mHasAudio = true;
if (oldRealTime != -1) {
int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs);
if (newRealTime >= oldRealTime) {
return OK;
}
}
mNextBufferItemMediaUs = -1;
onDrainVideo_l();
return OK;
}
void MediaSync::setName(const AString &name) {
Mutex::Autolock lock(mMutex);
mInput->setConsumerName(String8(name.c_str()));
}
status_t MediaSync::setVideoFrameRateHint(float rate) {
// ignored until we add the FrameScheduler
return rate >= 0.f ? OK : BAD_VALUE;
}
float MediaSync::getVideoFrameRate() {
// we don't know the frame rate
return -1.f;
}
status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
// validate settings
if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
|| syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
|| syncSettings.mTolerance < 0.f
|| syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
return BAD_VALUE;
}
Mutex::Autolock lock(mMutex);
// verify that we have the sync source
switch (syncSettings.mSource) {
case AVSYNC_SOURCE_AUDIO:
if (mAudioTrack == NULL) {
ALOGE("setSyncSettings: audio sync source requires an audio track");
return BAD_VALUE;
}
break;
case AVSYNC_SOURCE_VSYNC:
if (mOutput == NULL) {
ALOGE("setSyncSettings: vsync sync source requires an output surface");
return BAD_VALUE;
}
break;
default:
break;
}
mSyncSettings = syncSettings;
resync_l();
return OK;
}
void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
Mutex::Autolock lock(mMutex);
*syncSettings = mSyncSettings;
}
status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
Mutex::Autolock lock(mMutex);
status_t err = setPlaybackSettings_l(rate);
if (err == OK) {
// TODO: adjust rate if using VSYNC as source
updatePlaybackRate_l(rate.mSpeed);
}
return err;
}
status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
// We don't validate other audio settings.
// They will be validated when/if audiotrack is set.
return BAD_VALUE;
}
if (mAudioTrack != NULL) {
if (rate.mSpeed == 0.f) {
mAudioTrack->pause();
} else {
status_t err = mAudioTrack->setPlaybackRate(rate);
if (err != OK) {
return BAD_VALUE;
}
// ignore errors
(void)mAudioTrack->start();
}
}
mPlaybackSettings = rate;
return OK;
}
void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
Mutex::Autolock lock(mMutex);
*rate = mPlaybackSettings;
}
int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
int64_t realUs;
if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
// If failed to get current position, e.g. due to audio clock is
// not ready, then just play out video immediately without delay.
return nowUs;
}
return realUs;
}
int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) {
return (numFrames * 1000000LL / mNativeSampleRateInHz);
}
int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
CHECK(mAudioTrack != NULL);
uint32_t numFramesPlayed;
int64_t numFramesPlayedAt;
AudioTimestamp ts;
static const int64_t kStaleTimestamp100ms = 100000;
status_t res = mAudioTrack->getTimestamp(ts);
if (res == OK) {
// case 1: mixing audio tracks.
numFramesPlayed = ts.mPosition;
numFramesPlayedAt =
ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
const int64_t timestampAge = nowUs - numFramesPlayedAt;
if (timestampAge > kStaleTimestamp100ms) {
// This is an audio FIXME.
// getTimestamp returns a timestamp which may come from audio
// mixing threads. After pausing, the MixerThread may go idle,
// thus the mTime estimate may become stale. Assuming that the
// MixerThread runs 20ms, with FastMixer at 5ms, the max latency
// should be about 25ms with an average around 12ms (to be
// verified). For safety we use 100ms.
ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) "
"numFramesPlayedAt(%lld)",
(long long)nowUs, (long long)numFramesPlayedAt);
numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
}
//ALOGD("getTimestamp: OK %d %lld",
// numFramesPlayed, (long long)numFramesPlayedAt);
} else if (res == WOULD_BLOCK) {
// case 2: transitory state on start of a new track
numFramesPlayed = 0;
numFramesPlayedAt = nowUs;
//ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
// numFramesPlayed, (long long)numFramesPlayedAt);
} else {
// case 3: transitory at new track or audio fast tracks.
res = mAudioTrack->getPosition(&numFramesPlayed);
CHECK_EQ(res, (status_t)OK);
numFramesPlayedAt = nowUs;
numFramesPlayedAt += 1000LL * mAudioTrack->latency() / 2; /* XXX */
//ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt);
}
//can't be negative until 12.4 hrs, test.
//CHECK_EQ(numFramesPlayed & (1 << 31), 0);
int64_t durationUs =
getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed)
+ nowUs - numFramesPlayedAt;
if (durationUs < 0) {
// Occurs when numFramesPlayed position is very small and the following:
// (1) In case 1, the time nowUs is computed before getTimestamp() is
// called and numFramesPlayedAt is greater than nowUs by time more
// than numFramesPlayed.
// (2) In case 3, using getPosition and adding mAudioTrack->latency()
// to numFramesPlayedAt, by a time amount greater than
// numFramesPlayed.
//
// Both of these are transitory conditions.
ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld "
"set to zero", (long long)durationUs);
durationUs = 0;
}
ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) "
"framesAt(%lld)",
(long long)durationUs, (long long)nowUs, numFramesPlayed,
(long long)numFramesPlayedAt);
return durationUs;
}
void MediaSync::onDrainVideo_l() {
if (!isPlaying()) {
return;
}
while (!mBufferItems.empty()) {
int64_t nowUs = ALooper::GetNowUs();
BufferItem *bufferItem = &*mBufferItems.begin();
int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
if (itemRealUs <= nowUs) {
if (mHasAudio) {
if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
renderOneBufferItem_l(*bufferItem);
} else {
// too late.
returnBufferToInput_l(
bufferItem->mGraphicBuffer, bufferItem->mFence);
}
} else {
// always render video buffer in video-only mode.
renderOneBufferItem_l(*bufferItem);
// smooth out videos >= 10fps
mMediaClock->updateAnchor(
itemMediaUs, nowUs, itemMediaUs + 100000);
}
mBufferItems.erase(mBufferItems.begin());
mNextBufferItemMediaUs = -1;
} else {
if (mNextBufferItemMediaUs == -1
|| mNextBufferItemMediaUs > itemMediaUs) {
sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
msg->post(itemRealUs - nowUs);
mNextBufferItemMediaUs = itemMediaUs;
}
break;
}
}
}
void MediaSync::onFrameAvailableFromInput() {
Mutex::Autolock lock(mMutex);
// If there are too many outstanding buffers, wait until a buffer is
// released back to the input in onBufferReleased.
while (mNumOutstandingBuffers >= MAX_OUTSTANDING_BUFFERS) {
mReleaseCondition.wait(mMutex);
// If the sync is abandoned while we are waiting, the release
// condition variable will be broadcast, and we should just return
// without attempting to do anything more (since the input queue will
// also be abandoned).
if (mIsAbandoned) {
return;
}
}
++mNumOutstandingBuffers;
// Acquire and detach the buffer from the input.
BufferItem bufferItem;
status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */);
if (status != NO_ERROR) {
ALOGE("acquiring buffer from input failed (%d)", status);
return;
}
ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId());
status = mInput->detachBuffer(bufferItem.mBuf);
if (status != NO_ERROR) {
ALOGE("detaching buffer from input failed (%d)", status);
if (status == NO_INIT) {
// If the input has been abandoned, move on.
onAbandoned_l(true /* isInput */);
}
return;
}
if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
// Something is wrong since this buffer should be at our hands, bail.
mInput->consumerDisconnect();
onAbandoned_l(true /* isInput */);
return;
}
mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
mBufferItems.push_back(bufferItem);
if (mBufferItems.size() == 1) {
onDrainVideo_l();
}
}
void MediaSync::renderOneBufferItem_l( const BufferItem &bufferItem) {
IGraphicBufferProducer::QueueBufferInput queueInput(
bufferItem.mTimestamp,
bufferItem.mIsAutoTimestamp,
bufferItem.mDataSpace,
bufferItem.mCrop,
static_cast<int32_t>(bufferItem.mScalingMode),
bufferItem.mTransform,
bufferItem.mIsDroppable,
bufferItem.mFence);
// Attach and queue the buffer to the output.
int slot;
status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer);
ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status);
if (status == NO_ERROR) {
IGraphicBufferProducer::QueueBufferOutput queueOutput;
status = mOutput->queueBuffer(slot, queueInput, &queueOutput);
ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status);
}
if (status != NO_ERROR) {
returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence);
if (status == NO_INIT) {
// If the output has been abandoned, move on.
onAbandoned_l(false /* isInput */);
}
return;
}
ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
}
void MediaSync::onBufferReleasedByOutput() {
Mutex::Autolock lock(mMutex);
sp<GraphicBuffer> buffer;
sp<Fence> fence;
status_t status = mOutput->detachNextBuffer(&buffer, &fence);
ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
if (status == NO_INIT) {
// If the output has been abandoned, we can't do anything else,
// since buffer is invalid.
onAbandoned_l(false /* isInput */);
return;
}
ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
// If we've been abandoned, we can't return the buffer to the input, so just
// move on.
if (mIsAbandoned) {
return;
}
returnBufferToInput_l(buffer, fence);
}
void MediaSync::returnBufferToInput_l(
const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) {
ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
if (ix < 0) {
// The buffer is unknown, something is wrong, bail.
mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
onAbandoned_l(false /* isInput */);
return;
}
sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix);
mBuffersFromInput.removeItemsAt(ix);
// Attach and release the buffer back to the input.
int consumerSlot;
status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer);
ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status);
if (status == NO_ERROR) {
status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */,
EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence);
ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status);
}
if (status != NO_ERROR) {
// TODO: do we need to try to return this buffer later?
return;
}
ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId());
// Notify any waiting onFrameAvailable calls.
--mNumOutstandingBuffers;
mReleaseCondition.signal();
}
void MediaSync::onAbandoned_l(bool isInput) {
ALOGE("the %s has abandoned me", (isInput ? "input" : "output"));
if (!mIsAbandoned) {
if (isInput) {
mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
} else {
mInput->consumerDisconnect();
}
mIsAbandoned = true;
}
mReleaseCondition.broadcast();
}
void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
switch (msg->what()) {
case kWhatDrainVideo:
{
Mutex::Autolock lock(mMutex);
if (mNextBufferItemMediaUs != -1) {
int64_t nowUs = ALooper::GetNowUs();
int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs);
// The message could arrive earlier than expected due to
// various reasons, e.g., media clock has been changed because
// of new anchor time or playback rate. In such cases, the
// message needs to be re-posted.
if (itemRealUs > nowUs) {
msg->post(itemRealUs - nowUs);
break;
}
}
onDrainVideo_l();
break;
}
default:
TRESPASS();
break;
}
}
MediaSync::InputListener::InputListener(const sp<MediaSync> &sync)
: mSync(sync) {}
MediaSync::InputListener::~InputListener() {}
void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) {
mSync->onFrameAvailableFromInput();
}
// We don't care about sideband streams, since we won't relay them.
void MediaSync::InputListener::onSidebandStreamChanged() {
ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly.");
}
void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) {
Mutex::Autolock lock(mSync->mMutex);
mSync->onAbandoned_l(true /* isInput */);
}
MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync)
: mSync(sync) {}
MediaSync::OutputListener::~OutputListener() {}
void MediaSync::OutputListener::onBufferReleased() {
mSync->onBufferReleasedByOutput();
}
void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) {
Mutex::Autolock lock(mSync->mMutex);
mSync->onAbandoned_l(false /* isInput */);
}
} // namespace android