commit c52ab47449d3c6bf2af3668c2c753d0a33404a9a
author Aayush Soni <aayush.soni@ittiam.com> Fri Jun 18 19:16:34 2021 +0530
committer Ray Essick <essick@google.com> Wed Sep 08 08:54:32 2021 -0700
tree 2a63a5e33d000b50bcc96f5eacab75445ca4d350
parent 547282798ce5fe06b36f623219e799336a234ec9

mpeg4enc: update resolution check

Updated resolution check to limit encoder to 2048x2048

Bug: 190435883

Test: POC in bug descriptions

Change-Id: I749a9a3fda0924c17b3a4ddfb1c6402d2ce8552f

media/codecs/m4v_h263/enc/src/mp4enc_api.cpp

diff --git a/media/codecs/m4v_h263/enc/src/mp4enc_api.cpp b/media/codecs/m4v_h263/enc/src/mp4enc_api.cpp
index 234faef..00b2ab6 100644
--- a/media/codecs/m4v_h263/enc/src/mp4enc_api.cpp
+++ b/media/codecs/m4v_h263/enc/src/mp4enc_api.cpp
@@ -610,18 +610,19 @@
     /* Find the maximum width*height for memory allocation of the VOPs */
     for (idx = 0; idx < nLayers; idx++)
     {
-        temp_w = video->encParams->LayerWidth[idx];
-        temp_h = video->encParams->LayerHeight[idx];
+        temp_w = ((video->encParams->LayerWidth[idx] + 15) >> 4) << 4;
+        temp_h = ((video->encParams->LayerHeight[idx] + 15) >> 4) << 4;
+
+        if (temp_w > 2048 || temp_h > 2048) {
+            goto CLEAN_UP;
+        }
 
         if ((temp_w*temp_h) > max)
         {
             max = temp_w * temp_h;
-            max_width = ((temp_w + 15) >> 4) << 4;
-            max_height = ((temp_h + 15) >> 4) << 4;
-            if (((uint64_t)max_width * max_height) > (uint64_t)INT32_MAX
-                    || temp_w > INT32_MAX - 15 || temp_h > INT32_MAX - 15) {
-                goto CLEAN_UP;
-            }
+            max_width = temp_w;
+            max_height = temp_h;
+
             nTotalMB = ((max_width * max_height) >> 8);
         }