commit 5480c1d2817b9a564e69d3c0c151c6411ae4909b
author Jorge E. Moreira <jemoreira@google.com> Wed Jun 30 11:33:51 2021 -0700
committer Jorge Moreira Broche <jemoreira@google.com> Mon Jul 12 19:51:34 2021 +0000
tree 2bb892d71caf976167223d81e692319771f72860
parent 959630e21a4893deb30e6dc6a1b6efbfa04a1d8b

Avoid overflow in calc_iframe_target_size

The changed product was observed to attempt to multiply 1800 by 2500000,
which overflows unsigned 32 bits. Converting to unsigned 64 bits first
and testing whether the final result fits in 32 bits solves the problem.

Fixes: 179686142
Test: run cuttlefish locally for over 30 min without observing the issue
Change-Id: I5d27317bf14b0311b739144c451d8e172db01945

README.version

diff --git a/README.version b/README.version
index 7e8c25d..f555bcc 100644
--- a/README.version
+++ b/README.version
@@ -11,3 +11,4 @@
   5e065cf9d vp8/{ratectrl,onyx_if}: fix some signed integer overflows
   5eab093a7 vp9_ratectrl: fix some signed integer overflows
   baefbe85d Cap target bitrate to raw rate internally
+  5f345a924 Avoid overflow in calc_iframe_target_size

libvpx/vp8/encoder/ratectrl.c

diff --git a/libvpx/vp8/encoder/ratectrl.c b/libvpx/vp8/encoder/ratectrl.c
index d8d55fc..d93fb59 100644
--- a/libvpx/vp8/encoder/ratectrl.c
+++ b/libvpx/vp8/encoder/ratectrl.c
@@ -349,8 +349,12 @@
   }
 
   if (cpi->oxcf.rc_max_intra_bitrate_pct) {
-    unsigned int max_rate =
-        cpi->per_frame_bandwidth * cpi->oxcf.rc_max_intra_bitrate_pct / 100;
+    unsigned int max_rate;
+    // This product may overflow unsigned int
+    uint64_t product = cpi->per_frame_bandwidth;
+    product *= cpi->oxcf.rc_max_intra_bitrate_pct;
+    product /= 100;
+    max_rate = (unsigned int)VPXMIN(INT_MAX, product);
 
     if (target > max_rate) target = max_rate;
   }