am 479b4de0: SoftAVCEnc: check requested memory size before allocation.
* commit '479b4de0d267eb7d4c419f4da0069186a952ad17':
SoftAVCEnc: check requested memory size before allocation.
diff --git a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
index 4a21a3e..4cdf8c4 100644
--- a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
+++ b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
@@ -257,6 +257,10 @@
if (mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {
// Color conversion is needed.
CHECK(mInputFrameData == NULL);
+ if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+ ALOGE("Buffer size is too big.");
+ return OMX_ErrorUndefined;
+ }
mInputFrameData =
(uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);
CHECK(mInputFrameData != NULL);
@@ -278,6 +282,10 @@
int32_t nMacroBlocks = ((((mVideoWidth + 15) >> 4) << 4) *
(((mVideoHeight + 15) >> 4) << 4)) >> 8;
CHECK(mSliceGroup == NULL);
+ if (nMacroBlocks > SIZE_MAX / sizeof(uint32_t)) {
+ ALOGE("requested memory size is too big.");
+ return OMX_ErrorUndefined;
+ }
mSliceGroup = (uint32_t *) malloc(sizeof(uint32_t) * nMacroBlocks);
CHECK(mSliceGroup != NULL);
for (int ii = 0, idx = 0; ii < nMacroBlocks; ++ii) {
@@ -698,6 +706,10 @@
if (mStoreMetaDataInBuffers) {
mVideoColorFormat == OMX_COLOR_FormatYUV420SemiPlanar;
if (mInputFrameData == NULL) {
+ if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+ ALOGE("Buffer size is too big.");
+ return OMX_ErrorUndefined;
+ }
mInputFrameData =
(uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);
}
