commit b52c75787d068a92c961d29f973e975b7651abe5
author Pawin Vongmasa <pawin@google.com> Mon Jul 18 20:12:02 2016 -0700
committer gitbuildkicker <android-build@google.com> Thu Aug 25 21:56:08 2016 -0700
tree ce9839686958e5a7e0c88b4db9cf7ed933dbd3f6
parent 6679b5088f36693f5708dcaedd0c9ab7c66df27c

SoftMPEG4: Check the buffer size before writing the reference frame.

Also prevent overflow in SoftMPEG4 and division by zero in SoftMPEG4Encoder.

Bug: 30033990
Change-Id: I7701f5fc54c2670587d122330e5dc851f64ed3c2
(cherry picked from commit 695123195034402ca76169b195069c28c30342d3)

media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp

diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
index bb59ae4..e1cfc06 100644
--- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
+++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
@@ -210,8 +210,17 @@
             PortInfo *port = editPortInfo(1);
             OMX_BUFFERHEADERTYPE *outHeader = port->mBuffers.editItemAt(1).mHeader;
 
+            OMX_U32 yFrameSize = sizeof(uint8) * mHandle->size;
+            if ((outHeader->nAllocLen < yFrameSize) ||
+                    (outHeader->nAllocLen - yFrameSize < yFrameSize / 2)) {
+                ALOGE("Too small output buffer for reference frame: %zu bytes",
+                        outHeader->nAllocLen);
+                android_errorWriteLog(0x534e4554, "30033990");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
             PVSetReferenceYUV(mHandle, outHeader->pBuffer);
-
             mFramesConfigured = true;
         }
 
@@ -229,7 +238,16 @@
         int32_t bufferSize = inHeader->nFilledLen;
         int32_t tmp = bufferSize;
 
-        OMX_U32 frameSize = (mWidth * mHeight * 3) / 2;
+        OMX_U32 frameSize;
+        OMX_U64 yFrameSize = (OMX_U64)mWidth * (OMX_U64)mHeight;
+        if (yFrameSize > ((OMX_U64)UINT32_MAX / 3) * 2) {
+            ALOGE("Frame size too large");
+            notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+            mSignalledError = true;
+            return;
+        }
+        frameSize = (OMX_U32)(yFrameSize + (yFrameSize / 2));
+
         if (outHeader->nAllocLen < frameSize) {
             android_errorWriteLog(0x534e4554, "27833616");
             ALOGE("Insufficient output buffer size");

media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp

diff --git a/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp b/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
index 7638bb7..2eb51c9 100644
--- a/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
+++ b/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
@@ -116,6 +116,10 @@
         ALOGE("Failed to get default encoding parameters");
         return OMX_ErrorUndefined;
     }
+    if (mFramerate == 0) {
+        ALOGE("Framerate should not be 0");
+        return OMX_ErrorUndefined;
+    }
     mEncParams->encMode = mEncodeMode;
     mEncParams->encWidth[0] = mWidth;
     mEncParams->encHeight[0] = mHeight;