commit 42b873b2fac627fa16d654e5ffb979db02b0d44d
author Treehugger Robot <treehugger-gerrit@google.com> Mon Jul 12 20:59:01 2021 +0000
committer Gerrit Code Review <noreply-gerritcodereview@google.com> Mon Jul 12 20:59:01 2021 +0000
tree 4077c7c8d5ff7b58aadb1569b17677ca417bdad8
parent 97913d2610d2423654b48b8a56c357363a1d6768
parent 1e84c7734e13875c8d019d867ad980ee74ea341d

Merge "Avoid overflow in calc_iframe_target_size"

README.version

diff --git a/README.version b/README.version
index 7e8c25d..f555bcc 100644
--- a/README.version
+++ b/README.version
@@ -11,3 +11,4 @@
   5e065cf9d vp8/{ratectrl,onyx_if}: fix some signed integer overflows
   5eab093a7 vp9_ratectrl: fix some signed integer overflows
   baefbe85d Cap target bitrate to raw rate internally
+  5f345a924 Avoid overflow in calc_iframe_target_size

libvpx/vp8/encoder/ratectrl.c

diff --git a/libvpx/vp8/encoder/ratectrl.c b/libvpx/vp8/encoder/ratectrl.c
index d8d55fc..d93fb59 100644
--- a/libvpx/vp8/encoder/ratectrl.c
+++ b/libvpx/vp8/encoder/ratectrl.c
@@ -349,8 +349,12 @@
   }
 
   if (cpi->oxcf.rc_max_intra_bitrate_pct) {
-    unsigned int max_rate =
-        cpi->per_frame_bandwidth * cpi->oxcf.rc_max_intra_bitrate_pct / 100;
+    unsigned int max_rate;
+    // This product may overflow unsigned int
+    uint64_t product = cpi->per_frame_bandwidth;
+    product *= cpi->oxcf.rc_max_intra_bitrate_pct;
+    product /= 100;
+    max_rate = (unsigned int)VPXMIN(INT_MAX, product);
 
     if (target > max_rate) target = max_rate;
   }