Ogg: avoid size_t overflow in base64 decoding

Bug: 23707088
Change-Id: I8d32841fee3213c721cdcc57788807ea64d19d74

media/libstagefright/OggExtractor.cpp

diff --git a/media/libstagefright/OggExtractor.cpp b/media/libstagefright/OggExtractor.cpp
index 073c53f..28c9c7c 100644
--- a/media/libstagefright/OggExtractor.cpp
+++ b/media/libstagefright/OggExtractor.cpp
@@ -844,11 +844,14 @@
         }
     }
 
-    size_t outLen = 3 * size / 4 - padding;
-
-    *outSize = outLen;
+    // We divide first to avoid overflow. It's OK to do this because we
+    // already made sure that size % 4 == 0.
+    size_t outLen = (size / 4) * 3 - padding;
 
     void *buffer = malloc(outLen);
+    if (buffer == NULL) {
+        return NULL;
+    }
 
     uint8_t *out = (uint8_t *)buffer;
     size_t j = 0;
@@ -867,10 +870,10 @@
         } else if (c == '/') {
             value = 63;
         } else if (c != '=') {
-            return NULL;
+            break;
         } else {
             if (i < n - padding) {
-                return NULL;
+                break;
             }
 
             value = 0;
@@ -888,6 +891,13 @@
         }
     }
 
+    // Check if we exited the loop early.
+    if (j < outLen) {
+        free(buffer);
+        return NULL;
+    }
+
+    *outSize = outLen;
     return (uint8_t *)buffer;
 }