codecs: check OMX buffer size before use in (avc|hevc|mpeg2)dec
Bug: 27833616
Change-Id: Ic4045a3f56f53b08d0b1264b2a91b8f43e91b738
(cherry picked from commit 87fdee0bc9e3ac4d2a88ef0a8e150cfdf08c161d)
diff --git a/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp b/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp
index afbe230..2130ccf 100644
--- a/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp
+++ b/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp
@@ -381,7 +381,7 @@
resetPlugin();
}
-void SoftAVC::setDecodeArgs(
+bool SoftAVC::setDecodeArgs(
ivd_video_decode_ip_t *ps_dec_ip,
ivd_video_decode_op_t *ps_dec_op,
OMX_BUFFERHEADERTYPE *inHeader,
@@ -389,7 +389,6 @@
size_t timeStampIx) {
size_t sizeY = outputBufferWidth() * outputBufferHeight();
size_t sizeUV;
- uint8_t *pBuf;
ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t);
ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t);
@@ -409,22 +408,28 @@
ps_dec_ip->u4_num_Bytes = 0;
}
- if (outHeader) {
- pBuf = outHeader->pBuffer;
- } else {
- pBuf = mFlushOutBuffer;
- }
-
sizeUV = sizeY / 4;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
+ uint8_t *pBuf;
+ if (outHeader) {
+ if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) {
+ android_errorWriteLog(0x534e4554, "27569635");
+ return false;
+ }
+ pBuf = outHeader->pBuffer;
+ } else {
+ // mFlushOutBuffer always has the right size.
+ pBuf = mFlushOutBuffer;
+ }
+
ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf;
ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY;
ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV;
ps_dec_ip->s_out_buffer.u4_num_bufs = 3;
- return;
+ return true;
}
void SoftAVC::onPortFlushCompleted(OMX_U32 portIndex) {
/* Once the output buffers are flushed, ignore any buffers that are held in decoder */
@@ -573,7 +578,12 @@
WORD32 timeDelay, timeTaken;
size_t sizeY, sizeUV;
- setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx);
+ if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
+ ALOGE("Decoder arg setup failed");
+ notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+ mSignalledError = true;
+ return;
+ }
// If input dump is enabled, then write to file
DUMP_TO_FILE(mInFile, s_dec_ip.pv_stream_buffer, s_dec_ip.u4_num_Bytes);
diff --git a/media/libstagefright/codecs/avcdec/SoftAVCDec.h b/media/libstagefright/codecs/avcdec/SoftAVCDec.h
index 9dcabb4..c710c76 100644
--- a/media/libstagefright/codecs/avcdec/SoftAVCDec.h
+++ b/media/libstagefright/codecs/avcdec/SoftAVCDec.h
@@ -109,7 +109,7 @@
status_t resetPlugin();
- void setDecodeArgs(
+ bool setDecodeArgs(
ivd_video_decode_ip_t *ps_dec_ip,
ivd_video_decode_op_t *ps_dec_op,
OMX_BUFFERHEADERTYPE *inHeader,
diff --git a/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp b/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp
index e601125..a70755c 100644
--- a/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp
+++ b/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp
@@ -343,14 +343,13 @@
resetPlugin();
}
-void SoftHEVC::setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
+bool SoftHEVC::setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
ivd_video_decode_op_t *ps_dec_op,
OMX_BUFFERHEADERTYPE *inHeader,
OMX_BUFFERHEADERTYPE *outHeader,
size_t timeStampIx) {
size_t sizeY = outputBufferWidth() * outputBufferHeight();
size_t sizeUV;
- uint8_t *pBuf;
ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t);
ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t);
@@ -370,22 +369,28 @@
ps_dec_ip->u4_num_Bytes = 0;
}
- if (outHeader) {
- pBuf = outHeader->pBuffer;
- } else {
- pBuf = mFlushOutBuffer;
- }
-
sizeUV = sizeY / 4;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
+ uint8_t *pBuf;
+ if (outHeader) {
+ if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) {
+ android_errorWriteLog(0x534e4554, "27569635");
+ return false;
+ }
+ pBuf = outHeader->pBuffer;
+ } else {
+ // mFlushOutBuffer always has the right size.
+ pBuf = mFlushOutBuffer;
+ }
+
ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf;
ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY;
ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV;
ps_dec_ip->s_out_buffer.u4_num_bufs = 3;
- return;
+ return true;
}
void SoftHEVC::onPortFlushCompleted(OMX_U32 portIndex) {
/* Once the output buffers are flushed, ignore any buffers that are held in decoder */
@@ -520,7 +525,12 @@
WORD32 timeDelay, timeTaken;
size_t sizeY, sizeUV;
- setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx);
+ if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
+ ALOGE("Decoder arg setup failed");
+ notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+ mSignalledError = true;
+ return;
+ }
GETTIME(&mTimeStart, NULL);
/* Compute time elapsed between end of previous decode()
diff --git a/media/libstagefright/codecs/hevcdec/SoftHEVC.h b/media/libstagefright/codecs/hevcdec/SoftHEVC.h
index 21bb99e..943edfd 100644
--- a/media/libstagefright/codecs/hevcdec/SoftHEVC.h
+++ b/media/libstagefright/codecs/hevcdec/SoftHEVC.h
@@ -106,7 +106,7 @@
status_t resetDecoder();
status_t resetPlugin();
- void setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
+ bool setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
ivd_video_decode_op_t *ps_dec_op,
OMX_BUFFERHEADERTYPE *inHeader,
OMX_BUFFERHEADERTYPE *outHeader,
diff --git a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
index 4307c4e..e134d38 100644
--- a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
+++ b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
@@ -466,7 +466,7 @@
return ret;
}
-void SoftMPEG2::setDecodeArgs(
+bool SoftMPEG2::setDecodeArgs(
ivd_video_decode_ip_t *ps_dec_ip,
ivd_video_decode_op_t *ps_dec_op,
OMX_BUFFERHEADERTYPE *inHeader,
@@ -474,7 +474,6 @@
size_t timeStampIx) {
size_t sizeY = outputBufferWidth() * outputBufferHeight();
size_t sizeUV;
- uint8_t *pBuf;
ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t);
ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t);
@@ -494,22 +493,28 @@
ps_dec_ip->u4_num_Bytes = 0;
}
- if (outHeader) {
- pBuf = outHeader->pBuffer;
- } else {
- pBuf = mFlushOutBuffer;
- }
-
sizeUV = sizeY / 4;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
+ uint8_t *pBuf;
+ if (outHeader) {
+ if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) {
+ android_errorWriteLog(0x534e4554, "27569635");
+ return false;
+ }
+ pBuf = outHeader->pBuffer;
+ } else {
+ // mFlushOutBuffer always has the right size.
+ pBuf = mFlushOutBuffer;
+ }
+
ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf;
ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY;
ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV;
ps_dec_ip->s_out_buffer.u4_num_bufs = 3;
- return;
+ return true;
}
void SoftMPEG2::onPortFlushCompleted(OMX_U32 portIndex) {
/* Once the output buffers are flushed, ignore any buffers that are held in decoder */
@@ -622,7 +627,11 @@
WORD32 timeDelay, timeTaken;
size_t sizeY, sizeUV;
- setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx);
+ if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
+ ALOGE("Decoder arg setup failed");
+ notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+ return;
+ }
// If input dump is enabled, then write to file
DUMP_TO_FILE(mInFile, s_dec_ip.pv_stream_buffer, s_dec_ip.u4_num_Bytes);
@@ -665,9 +674,9 @@
CHECK_EQ(reInitDecoder(), (status_t)OK);
- setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx);
-
- ivdec_api_function(mCodecCtx, (void *)&s_dec_ip, (void *)&s_dec_op);
+ if (setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
+ ivdec_api_function(mCodecCtx, (void *)&s_dec_ip, (void *)&s_dec_op);
+ }
return;
}
diff --git a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
index a625e08..f48b70b 100644
--- a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
+++ b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
@@ -117,7 +117,7 @@
status_t resetPlugin();
status_t reInitDecoder();
- void setDecodeArgs(
+ bool setDecodeArgs(
ivd_video_decode_ip_t *ps_dec_ip,
ivd_video_decode_op_t *ps_dec_op,
OMX_BUFFERHEADERTYPE *inHeader,
