commit 3fcf95935950ad3fc4bf37eee8fb6e4c77c62339
author Ram Mohan M <ram.mohan@ittiam.com> Wed Sep 27 19:57:45 2023 +0530
committer Harish Mahendrakar <harish.mahendrakar@ittiam.com> Wed Sep 27 13:57:19 2023 -0700
tree 82c6b3950512279185a62a9cfd0be2f2f820add8
parent 2ff6f153766723731d9eb101ef8b2373356d638f

libavcenc: relocate the update post enc routines

During bitstream overflow errors, it is possible that update post
enc gets skipped as last row might never have to be entropy coded
as we have already consumed the buffer. This can be fatal for
encoding further frames. Move this update to a location where it
is guaranteed to be updated

Bug: oss-fuzz-62378
Test: ./avc_enc_fuzzer

Change-Id: I279deb1f30c15c057c0b6e909834313a1df4fdac

encoder/ih264e_encode.c

diff --git a/encoder/ih264e_encode.c b/encoder/ih264e_encode.c
index 1db0850..275bb59 100644
--- a/encoder/ih264e_encode.c
+++ b/encoder/ih264e_encode.c
@@ -586,6 +586,12 @@
 
         ih264_list_reset(ps_codec->pv_entropy_jobq);
 
+        error_status = ih264e_update_rc_post_enc(ps_codec, ctxt_sel, (ps_codec->i4_poc == 0));
+        SET_ERROR_ON_RETURN(error_status,
+                            ((error_status == IH264E_BITSTREAM_BUFFER_OVERFLOW) ?
+                                            IVE_UNSUPPORTEDPARAM : IVE_FATALERROR),
+                            ps_video_encode_op->s_ive_op.u4_error_code, IV_FAIL);
+
         if (ps_codec->s_cfg.u4_enable_quality_metrics & QUALITY_MASK_PSNR)
         {
             ih264e_compute_quality_stats(ps_proc);

encoder/ih264e_process.c

diff --git a/encoder/ih264e_process.c b/encoder/ih264e_process.c
index a0a04e0..769e124 100644
--- a/encoder/ih264e_process.c
+++ b/encoder/ih264e_process.c
@@ -2621,12 +2621,6 @@
                 /* entropy code all mbs enlisted under the current job */
                 error_status = ih264e_entropy(ps_proc);
 
-                if (s_job.i2_mb_y == ps_proc->i4_ht_mbs - 1)
-                {
-                    error_status |= ih264e_update_rc_post_enc(ps_codec, ctxt_sel,
-                                                              (ps_codec->i4_poc == 0));
-                }
-
                 /* Dont execute any further instructions until store synchronization took place */
                 DATA_SYNC();