commit acdec314f907a4eb78ab17c62feb31b0b53a532d
author Wan-Teh Chang <wtc@google.com> Mon Apr 08 19:00:12 2024 -0700
committer Jerome Jiang <jianj@google.com> Mon Apr 22 14:47:56 2024 +0000
tree c3cacc0a8defd3d05e2997b47c646f92d20315c2
parent 8156fb76d88845d716867d20333fd27001be47a8

Apply stride_align to byte count, not pixel count

stride_align is documented to be the "alignment, in bytes, of each row
in the image (stride)."

Change-Id: I4663f2fdf264800a0b8441772749920780248fbe
(cherry picked from commit a38ab61907e7619c6b100a769284505959def53e)

aom/src/aom_image.c

diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c
index e10b8a9..09b6dd4 100644
--- a/aom/src/aom_image.c
+++ b/aom/src/aom_image.c
@@ -119,8 +119,9 @@
   assert(d_h <= h);
 
   uint64_t s = (fmt & AOM_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / bit_depth;
-  s = (s + 2 * border + stride_align - 1) & ~((uint64_t)stride_align - 1);
+  s = s + 2 * border;
   s = s * bit_depth / 8;
+  s = (s + stride_align - 1) & ~((uint64_t)stride_align - 1);
   if (s > INT_MAX) goto fail;
   const int stride_in_bytes = (int)s;