commit 76dea1de8c92588358923300cbbc038144d302eb
author Wei Jia <wjia@google.com> Fri Aug 21 16:49:51 2015 -0700
committer Wei Jia <wjia@google.com> Fri Aug 21 16:49:51 2015 -0700
tree 04e770c6c7fc7cea5b79f98515bebb13f94d5ee5
parent e857d28d48e6088d4cf5fcbed7f411c73622b388

libstagefright: check overflow before memory allocation in OMXCodec.cpp

Bug: 23416608
Change-Id: I4dacd38ed42db8f4887c3ee386dc909451f4346f

media/libstagefright/OMXCodec.cpp

diff --git a/media/libstagefright/OMXCodec.cpp b/media/libstagefright/OMXCodec.cpp
index 96aa808b..b1dde80 100644
--- a/media/libstagefright/OMXCodec.cpp
+++ b/media/libstagefright/OMXCodec.cpp
@@ -1658,6 +1658,9 @@
             def.nBufferCountActual, def.nBufferSize,
             portIndex == kPortIndexInput ? "input" : "output");
 
+    if (def.nBufferSize != 0 && def.nBufferCountActual > SIZE_MAX / def.nBufferSize) {
+        return BAD_VALUE;
+    }
     size_t totalSize = def.nBufferCountActual * def.nBufferSize;
     mDealer[portIndex] = new MemoryDealer(totalSize, "OMXCodec");