Issue 2071329: audio track is shorter than video track for video capture on sholes
Add API to retrieve number of frames dropped by audio input kernel driver.
Submitted on behalf of Masaki Sato <masaki.sato@motorola.com>
diff --git a/android/author/android_audio_input.cpp b/android/author/android_audio_input.cpp
index 81e121e..5e030f0 100644
--- a/android/author/android_audio_input.cpp
+++ b/android/author/android_audio_input.cpp
@@ -23,6 +23,7 @@
#include "android_audio_input.h"
#include "pv_mime_string_utils.h"
#include "oscl_dll.h"
+#include "pvmf_duration_infomessage.h"
#include <media/AudioRecord.h>
#include <sys/prctl.h>
@@ -1173,6 +1174,19 @@
iOSSRequestQueue.erase(&iOSSRequestQueue[0]);
iOSSRequestQueueLock.Unlock();
+ uint32 inputFramesLost = record->getInputFramesLost();
+ if (inputFramesLost > 0) {
+ uint32 dataDurationLost = inputFramesLost * 1000 / iAudioSamplingRate;
+ LOGW("Notifying %d observers of %u ms of lost audio", iObservers.size(), dataDurationLost);
+
+ for(uint32 i = 0; i < iObservers.size(); i++) {
+ int32 leavecode = 0;
+ PVMFDurationInfoMessage* eventMsg = NULL;
+ OSCL_TRY(leavecode, eventMsg = OSCL_NEW(PVMFDurationInfoMessage, (dataDurationLost)));
+ iObservers[i]->ReportInfoEvent(PVMFInfoOverflow, OSCL_STATIC_CAST(PVInterface*, eventMsg));
+ }
+ }
+
int numOfBytes = record->read(data, kBufferSize);
//LOGV("read %d bytes", numOfBytes);
if (numOfBytes <= 0)
diff --git a/android/author/android_camera_input.cpp b/android/author/android_camera_input.cpp
index 379b708..7f2c37c 100644
--- a/android/author/android_camera_input.cpp
+++ b/android/author/android_camera_input.cpp
@@ -53,6 +53,7 @@
iAuthorClock(NULL),
iClockNotificationsInf(NULL),
iAudioFirstFrameTs(0),
+ iAudioLossDuration(0),
pPmemInfo(NULL)
{
LOGV("constructor(%p)", this);
@@ -72,6 +73,7 @@
mFlags = 0;
iFrameQueue.reserve(5);
iFrameQueueMutex.Create();
+ iAudioLossMutex.Create();
// setup callback listener
mListener = new AndroidCameraInputListener(this);
@@ -121,6 +123,7 @@
mCamera.clear();
}
iFrameQueueMutex.Close();
+ iAudioLossMutex.Close();
mListener.clear();
}
@@ -423,7 +426,7 @@
#endif
// View finder freeze detection
// Note for low frame rate, we don't bother to log view finder freezes
- int processingTimeInMs = (systemTime()/1000000L - iAudioFirstFrameTs) - data.iXferHeader.timestamp;
+ int processingTimeInMs = (systemTime()/1000000L - (iAudioFirstFrameTs + iAudioLossDuration)) - data.iXferHeader.timestamp;
if (processingTimeInMs >= VIEW_FINDER_FREEZE_DETECTION_TOLERANCE && mFrameRate >= 10.0) {
LOGW("Frame %p takes too long (%d ms) to process, staring at %d", data.iFrameBuffer.get(), processingTimeInMs, iAudioFirstFrameTs);
}
@@ -992,6 +995,7 @@
// Remove and destroy the clock state observer
RemoveDestroyClockObs();
iDataEventCounter = 0;
+ iAudioLossDuration = 0;
iWriteState = EWriteOK;
if ( (iState == STATE_STARTED) || (iState == STATE_PAUSED) ) {
if (mCamera != NULL) {
@@ -1184,13 +1188,24 @@
if (iAudioFirstFrameTs == 0)
iAudioFirstFrameTs = ts;
- if (ts < iAudioFirstFrameTs) {
+ iAudioLossMutex.Lock();
+ if (ts < iAudioFirstFrameTs + iAudioLossDuration) {
// Drop the frame
+ iAudioLossMutex.Unlock();
mCamera->releaseRecordingFrame(frame);
return PVMFSuccess;
} else {
- // calculate timestamp as offset from start time
- ts -= iAudioFirstFrameTs;
+ // calculate timestamp as offset from start time plus lost audio
+ ts -= (iAudioFirstFrameTs + iAudioLossDuration);
+ }
+ iAudioLossMutex.Unlock();
+
+ // Determine how much video to drop so when we resync to the audio
+ // time we don't go into old video (we need increasing timestamps)
+ if (ts <= iTimeStamp) {
+ LOGD("Dropping video frame to catch up for audio ts %lu, dropUntil %lu", ts, iTimeStamp);
+ mCamera->releaseRecordingFrame(frame);
+ return PVMFSuccess;
}
// Make sure that no two samples have the same timestamp
@@ -1249,6 +1264,15 @@
return PVMFSuccess;
}
+// Value is expected to be in ms
+void AndroidCameraInput::setAudioLossDuration(uint32 duration)
+{
+ iAudioLossMutex.Lock();
+ LOGD("Update for lost audio for %lu for existing duration %lu", duration, iAudioLossDuration);
+ iAudioLossDuration += duration;
+ iAudioLossMutex.Unlock();
+}
+
// camera callback interface
void AndroidCameraInputListener::postData(int32_t msgType, const sp<IMemory>& dataPtr)
{
@@ -1290,4 +1314,3 @@
}
-
diff --git a/android/author/android_camera_input.h b/android/author/android_camera_input.h
index ba83a54..6ddccc3 100644
--- a/android/author/android_camera_input.h
+++ b/android/author/android_camera_input.h
@@ -410,6 +410,7 @@
// add for Camcorder
PVMFStatus postWriteAsync(nsecs_t timestamp, const sp<IMemory>& frame);
+ void setAudioLossDuration(uint32 duration);
bool isRecorderStarting() { return iState==STATE_STARTED?true:false; }
@@ -531,6 +532,8 @@
PVMFMediaClockNotificationsInterface *iClockNotificationsInf;
uint32 iAudioFirstFrameTs;
+ OsclMutex iAudioLossMutex;
+ uint32 iAudioLossDuration;
PVRefBufferAlloc mbufferAlloc;
// data structures for tunneling buffers
diff --git a/android/author/authordriver.cpp b/android/author/authordriver.cpp
index fd81766..7a61439 100644
--- a/android/author/authordriver.cpp
+++ b/android/author/authordriver.cpp
@@ -27,6 +27,7 @@
#include "pv_omxcore.h"
#include <sys/prctl.h>
#include "pvmf_composer_size_and_duration.h"
+#include "pvmf_duration_infomessage.h"
#include "android_camera_input.h"
using namespace android;
@@ -1407,10 +1408,28 @@
LOGV("HandleInformationalEvent(%d)", event_type);
}
- mListener->notify(
- MEDIA_RECORDER_EVENT_INFO,
- GetMediaRecorderInfoCode(aEvent),
- aEvent.GetEventType());
+ if (PVMFInfoOverflow == event_type && mVideoInputMIO) {
+ PVUuid infomsguuid = PVMFDurationInfoMessageInterfaceUUID;
+ PVMFDurationInfoMessageInterface* eventMsg = NULL;
+ PVInterface* infoExtInterface = aEvent.GetEventExtensionInterface();
+ if (infoExtInterface &&
+ infoExtInterface->queryInterface(infomsguuid, (PVInterface*&)eventMsg) && eventMsg) {
+ PVUuid eventuuid;
+ int32 infoCode;
+ eventMsg->GetCodeUUID(infoCode, eventuuid);
+ if (eventuuid == infomsguuid) {
+ uint32 duration = eventMsg->GetDuration();
+ ((AndroidCameraInput *)mVideoInputMIO)->setAudioLossDuration(duration);
+ }
+ eventMsg->removeRef();
+ eventMsg = NULL;
+ }
+ } else {
+ mListener->notify(
+ MEDIA_RECORDER_EVENT_INFO,
+ GetMediaRecorderInfoCode(aEvent),
+ aEvent.GetEventType());
+ }
}
status_t AuthorDriver::setListener(const sp<IMediaPlayerClient>& listener) {
diff --git a/engines/author/src/pvauthorengine.cpp b/engines/author/src/pvauthorengine.cpp
index 4c790ba..5de9c42 100644
--- a/engines/author/src/pvauthorengine.cpp
+++ b/engines/author/src/pvauthorengine.cpp
@@ -522,6 +522,14 @@
PushCmdInFront(cmd);
}
break;
+
+ case PVMFInfoOverflow:
+ {
+ PVAsyncInformationalEvent event(aEvent.GetEventType(), NULL, aEvent.GetEventExtensionInterface());
+ iInfoEventObserver->HandleInformationalEvent(event);
+ }
+ break;
+
case PVMF_COMPOSER_FILESIZE_PROGRESS:
case PVMF_COMPOSER_DURATION_PROGRESS:
{
diff --git a/nodes/pvmediainputnode/src/pvmf_media_input_node.cpp b/nodes/pvmediainputnode/src/pvmf_media_input_node.cpp
index eb232d9..e5ba28e 100644
--- a/nodes/pvmediainputnode/src/pvmf_media_input_node.cpp
+++ b/nodes/pvmediainputnode/src/pvmf_media_input_node.cpp
@@ -560,8 +560,12 @@
////////////////////////////////////////////////////////////////////////////
OSCL_EXPORT_REF void PvmfMediaInputNode::ReportInfoEvent(PVMFEventType aEventType, PVInterface* aExtMsg)
{
- OSCL_UNUSED_ARG(aEventType);
- OSCL_UNUSED_ARG(aExtMsg);
+ if (aEventType == PVMFInfoOverflow) {
+ PVMFNodeInterface::ReportInfoEvent(aEventType, NULL, aExtMsg);
+ } else {
+ OSCL_UNUSED_ARG(aEventType);
+ OSCL_UNUSED_ARG(aExtMsg);
+ }
}
////////////////////////////////////////////////////////////////////////////
