SoftAVCEnc: check requested memory size before allocation.
Bug: 20674674
Change-Id: If80186a7b9078e575d389220f3bebe9f7630a956
(cherry picked from commit f6fe4340219a8e674f3250fe32d4697ec8184b24)
diff --git a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
index 24dfc29..0f56f4c 100644
--- a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
+++ b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
@@ -244,6 +244,10 @@
if (mColorFormat != OMX_COLOR_FormatYUV420Planar || mInputDataIsMeta) {
// Color conversion is needed.
free(mInputFrameData);
+ if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+ ALOGE("Buffer size is too big.");
+ return OMX_ErrorUndefined;
+ }
mInputFrameData =
(uint8_t *) malloc((mWidth * mHeight * 3 ) >> 1);
CHECK(mInputFrameData != NULL);
@@ -264,6 +268,10 @@
int32_t nMacroBlocks = divUp(mWidth, 16) * divUp(mHeight, 16);
CHECK(mSliceGroup == NULL);
+ if ((size_t)nMacroBlocks > SIZE_MAX / sizeof(uint32_t)) {
+ ALOGE("requested memory size is too big.");
+ return OMX_ErrorUndefined;
+ }
mSliceGroup = (uint32_t *) malloc(sizeof(uint32_t) * nMacroBlocks);
CHECK(mSliceGroup != NULL);
for (int ii = 0, idx = 0; ii < nMacroBlocks; ++ii) {
