commit 1b70db4be90e66fdd0473f34ad7bec69f269edeb
author James Zern <jzern@google.com> Tue Apr 26 19:05:56 2022 -0700
committer James Zern <jzern@google.com> Tue Apr 26 19:06:29 2022 -0700
tree c006cb57c1671d2fa665b179c2a9c22eab40a652
parent 19b45a26c62170c1fb0dfd18a083ddb84ef7e4a4

vp9: check postproc_state.limits allocs

Change-Id: I9d5df96580074375e4847d2e2f60a6a6d56eeea5

vp9/common/vp9_postproc.c

diff --git a/vp9/common/vp9_postproc.c b/vp9/common/vp9_postproc.c
index d2c8535..96519f0 100644
--- a/vp9/common/vp9_postproc.c
+++ b/vp9/common/vp9_postproc.c
@@ -360,6 +360,7 @@
     if (!cm->postproc_state.limits) {
       cm->postproc_state.limits =
           vpx_calloc(unscaled_width, sizeof(*cm->postproc_state.limits));
+      if (!cm->postproc_state.limits) return 1;
     }
   }
 

vp9/encoder/vp9_encoder.c

diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index ec6a756..89b7c8e 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -3684,9 +3684,9 @@
       case 6: l = 150; break;
     }
     if (!cpi->common.postproc_state.limits) {
-      cpi->common.postproc_state.limits =
-          vpx_calloc(cpi->un_scaled_source->y_width,
-                     sizeof(*cpi->common.postproc_state.limits));
+      CHECK_MEM_ERROR(cm, cpi->common.postproc_state.limits,
+                      vpx_calloc(cpi->un_scaled_source->y_width,
+                                 sizeof(*cpi->common.postproc_state.limits)));
     }
     vp9_denoise(&cpi->common, cpi->Source, cpi->Source, l,
                 cpi->common.postproc_state.limits);