Camera: Heic: Handle out-of-order buffer outputs
During reprocess usecases, the Camera HAL may return JPEG Appsegment
frames out of order. This leads to a scenario where
mAppSegmentConsumer->lockNextBuffer may not refer to the currently muxed
frame.
To work around this, the number of lockable appsegment buffers has been
increased.
Test: Vendor testing
Bug: 141169323
Change-Id: Ia498f6ddaba588b66cbaeee94fe8bdd7314bb90c
Merged-In: Ia498f6ddaba588b66cbaeee94fe8bdd7314bb90c
(cherry picked from commit b5986a36b2913ec100b110720c3e24e26fd0d1dd)
diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp
index 7663bb9..dc46285 100644
--- a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp
+++ b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp
@@ -60,12 +60,12 @@
mUseGrid(false),
mAppSegmentStreamId(-1),
mAppSegmentSurfaceId(-1),
- mAppSegmentBufferAcquired(false),
mMainImageStreamId(-1),
mMainImageSurfaceId(-1),
mYuvBufferAcquired(false),
mProducerListener(new ProducerListener()),
mDequeuedOutputBufferCnt(0),
+ mLockedAppSegmentBufferCnt(0),
mCodecOutputCounter(0),
mGridTimestampUs(0) {
}
@@ -132,7 +132,7 @@
sp<IGraphicBufferProducer> producer;
sp<IGraphicBufferConsumer> consumer;
BufferQueue::createBufferQueue(&producer, &consumer);
- mAppSegmentConsumer = new CpuConsumer(consumer, 1);
+ mAppSegmentConsumer = new CpuConsumer(consumer, kMaxAcquiredAppSegment);
mAppSegmentConsumer->setFrameAvailableListener(this);
mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));
mAppSegmentSurface = new Surface(producer);
@@ -527,12 +527,12 @@
mSettingsByTimestamp.erase(it);
}
- while (!mInputAppSegmentBuffers.empty() && !mAppSegmentBufferAcquired) {
+ while (!mInputAppSegmentBuffers.empty()) {
CpuConsumer::LockedBuffer imgBuffer;
auto it = mInputAppSegmentBuffers.begin();
auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer);
if (res == NOT_ENOUGH_DATA) {
- // Canot not lock any more buffers.
+ // Can not lock any more buffers.
break;
} else if ((res != OK) || (*it != imgBuffer.timestamp)) {
if (res != OK) {
@@ -542,6 +542,7 @@
ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64
" received buffer with time stamp: %" PRId64, __FUNCTION__,
*it, imgBuffer.timestamp);
+ mAppSegmentConsumer->unlockBuffer(imgBuffer);
}
mPendingInputFrames[*it].error = true;
mInputAppSegmentBuffers.erase(it);
@@ -553,7 +554,7 @@
mAppSegmentConsumer->unlockBuffer(imgBuffer);
} else {
mPendingInputFrames[imgBuffer.timestamp].appSegmentBuffer = imgBuffer;
- mAppSegmentBufferAcquired = true;
+ mLockedAppSegmentBufferCnt++;
}
mInputAppSegmentBuffers.erase(it);
}
@@ -563,7 +564,7 @@
auto it = mInputYuvBuffers.begin();
auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer);
if (res == NOT_ENOUGH_DATA) {
- // Canot not lock any more buffers.
+ // Can not lock any more buffers.
break;
} else if (res != OK) {
ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__,
@@ -797,12 +798,18 @@
}
}
- if (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0) {
- res = processCompletedInputFrame(timestamp, inputFrame);
- if (res != OK) {
- ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__,
- strerror(-res), res);
- return res;
+ if (inputFrame.pendingOutputTiles == 0) {
+ if (inputFrame.appSegmentWritten) {
+ res = processCompletedInputFrame(timestamp, inputFrame);
+ if (res != OK) {
+ ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__,
+ strerror(-res), res);
+ return res;
+ }
+ } else if (mLockedAppSegmentBufferCnt == kMaxAcquiredAppSegment) {
+ ALOGE("%s: Out-of-order app segment buffers reaches limit %u", __FUNCTION__,
+ kMaxAcquiredAppSegment);
+ return INVALID_OPERATION;
}
}
@@ -936,11 +943,17 @@
__FUNCTION__, strerror(-res), res);
return res;
}
- inputFrame.appSegmentWritten = true;
ALOGV("%s: [%" PRId64 "]: appSegmentSize is %zu, width %d, height %d, app1Size %zu",
__FUNCTION__, timestamp, appSegmentSize, inputFrame.appSegmentBuffer.width,
inputFrame.appSegmentBuffer.height, app1Size);
+
+ inputFrame.appSegmentWritten = true;
+ // Release the buffer now so any pending input app segments can be processed
+ mAppSegmentConsumer->unlockBuffer(inputFrame.appSegmentBuffer);
+ inputFrame.appSegmentBuffer.data = nullptr;
+ mLockedAppSegmentBufferCnt--;
+
return OK;
}
@@ -1095,7 +1108,6 @@
if (inputFrame->appSegmentBuffer.data != nullptr) {
mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer);
inputFrame->appSegmentBuffer.data = nullptr;
- mAppSegmentBufferAcquired = false;
}
while (!inputFrame->codecOutputBuffers.empty()) {
@@ -1133,11 +1145,13 @@
}
}
-void HeicCompositeStream::releaseInputFramesLocked(int64_t currentTs) {
+void HeicCompositeStream::releaseInputFramesLocked() {
auto it = mPendingInputFrames.begin();
while (it != mPendingInputFrames.end()) {
- if (it->first <= currentTs) {
- releaseInputFrameLocked(&it->second);
+ auto& inputFrame = it->second;
+ if (inputFrame.error ||
+ (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0)) {
+ releaseInputFrameLocked(&inputFrame);
it = mPendingInputFrames.erase(it);
} else {
it++;
@@ -1541,7 +1555,7 @@
// In case we landed in error state, return any pending buffers and
// halt all further processing.
compilePendingInputLocked();
- releaseInputFramesLocked(currentTs);
+ releaseInputFramesLocked();
return false;
}
@@ -1583,11 +1597,7 @@
mPendingInputFrames[currentTs].error = true;
}
- if (mPendingInputFrames[currentTs].error ||
- (mPendingInputFrames[currentTs].appSegmentWritten &&
- mPendingInputFrames[currentTs].pendingOutputTiles == 0)) {
- releaseInputFramesLocked(currentTs);
- }
+ releaseInputFramesLocked();
return true;
}
diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.h b/services/camera/libcameraservice/api2/HeicCompositeStream.h
index 9bec909..04e7b83 100644
--- a/services/camera/libcameraservice/api2/HeicCompositeStream.h
+++ b/services/camera/libcameraservice/api2/HeicCompositeStream.h
@@ -189,7 +189,7 @@
status_t processCompletedInputFrame(nsecs_t timestamp, InputFrame &inputFrame);
void releaseInputFrameLocked(InputFrame *inputFrame /*out*/);
- void releaseInputFramesLocked(int64_t currentTs);
+ void releaseInputFramesLocked();
size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize,
size_t* app1SegmentSize);
@@ -207,11 +207,13 @@
static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS);
static const android_dataspace kHeifDataSpace =
static_cast<android_dataspace>(HAL_DATASPACE_HEIF);
+ // Use the limit of pipeline depth in the API sepc as maximum number of acquired
+ // app segment buffers.
+ static const uint32_t kMaxAcquiredAppSegment = 8;
int mAppSegmentStreamId, mAppSegmentSurfaceId;
sp<CpuConsumer> mAppSegmentConsumer;
sp<Surface> mAppSegmentSurface;
- bool mAppSegmentBufferAcquired;
size_t mAppSegmentMaxSize;
CameraMetadata mStaticInfo;
@@ -232,6 +234,7 @@
// Keep all incoming APP segment Blob buffer pending further processing.
std::vector<int64_t> mInputAppSegmentBuffers;
+ int32_t mLockedAppSegmentBufferCnt;
// Keep all incoming HEIC blob buffer pending further processing.
std::vector<CodecOutputBufferInfo> mCodecOutputBuffers;
