commit 813201e174701130cca0e322647c366715e06a3f
author Jingning Han <jingning@google.com> Mon Jun 27 10:15:36 2016 -0700
committer Jingning Han <jingning@google.com> Mon Jun 27 12:46:07 2016 -0700
tree 39652fba12ab75b57161250bc9dcfc68b679eb60
parent cc16c5d8056feeaf22973c250b184d55017cd38a

Disable trellis optimized quantization in the first-pass

This resolves the use of uninitialized value in the first-pass
encoding.

Change-Id: I78bc19214a1bfde5c5641424550cbbe4e52cae99

vp10/encoder/encodemb.c

diff --git a/vp10/encoder/encodemb.c b/vp10/encoder/encodemb.c
index adcec9d..01e72c6 100644
--- a/vp10/encoder/encodemb.c
+++ b/vp10/encoder/encodemb.c
@@ -1201,7 +1201,7 @@
   MACROBLOCKD *const xd = &x->e_mbd;
   struct optimize_ctx ctx;
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
-  struct encode_b_args arg = {x, &ctx, &mbmi->skip, NULL, NULL};
+  struct encode_b_args arg = {x, &ctx, &mbmi->skip, NULL, NULL, 1};
   int plane;
 
   mbmi->skip = 1;
@@ -1262,7 +1262,7 @@
   MACROBLOCKD *const xd = &x->e_mbd;
   struct optimize_ctx ctx;
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
-  struct encode_b_args arg = {x, &ctx, &mbmi->skip, NULL, NULL};
+  struct encode_b_args arg = {x, &ctx, &mbmi->skip, NULL, NULL, 1};
   int plane;
 
   mbmi->skip = 1;
@@ -1338,16 +1338,22 @@
   l = &args->tl[blk_row];
   ctx = combine_entropy_contexts(*a, *l);
 
+  if (args->enable_optimize_b) {
 #if CONFIG_NEW_QUANT
-  vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize,
-                       tx_size, ctx);
+    vp10_xform_quant_nuq(x, plane, block, blk_row, blk_col, plane_bsize,
+                         tx_size, ctx);
 #else  // CONFIG_NEW_QUANT
-  vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-                   VP10_XFORM_QUANT_FP);
+    vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+                     VP10_XFORM_QUANT_FP);
 #endif  // CONFIG_NEW_QUANT
-  if (p->eobs[block]) {
-    *a = *l = vp10_optimize_b(x, plane, block, tx_size, ctx) > 0;
+    if (p->eobs[block]) {
+      *a = *l = vp10_optimize_b(x, plane, block, tx_size, ctx) > 0;
+    } else {
+      *a = *l = 0;
+    }
   } else {
+    vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+                     VP10_XFORM_QUANT_B);
     *a = *l = p->eobs[block] > 0;
   }
 
@@ -1378,7 +1384,8 @@
   ENTROPY_CONTEXT ta[2 * MAX_MIB_SIZE];
   ENTROPY_CONTEXT tl[2 * MAX_MIB_SIZE];
 
-  struct encode_b_args arg = {x, NULL, &xd->mi[0]->mbmi.skip, ta, tl};
+  struct encode_b_args arg = {x, NULL, &xd->mi[0]->mbmi.skip,
+                              ta, tl, enable_optimize_b};
   if (enable_optimize_b) {
     const struct macroblockd_plane* const pd = &xd->plane[plane];
     const TX_SIZE tx_size = plane ? get_uv_tx_size(&xd->mi[0]->mbmi, pd) :

vp10/encoder/encodemb.h

diff --git a/vp10/encoder/encodemb.h b/vp10/encoder/encodemb.h
index 90f37eb..8beb578 100644
--- a/vp10/encoder/encodemb.h
+++ b/vp10/encoder/encodemb.h
@@ -29,6 +29,7 @@
   int8_t *skip;
   ENTROPY_CONTEXT *ta;
   ENTROPY_CONTEXT *tl;
+  int8_t enable_optimize_b;
 };
 
 typedef enum VP10_XFORM_QUANT {

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 18a6011..708e8fc 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1208,9 +1208,8 @@
     return;
 
   if (!is_inter_block(mbmi)) {
-    struct encode_b_args intra_arg = {x, NULL, &mbmi->skip, args->t_above,
-                                      args->t_left};
-
+    struct encode_b_args intra_arg = {
+        x, NULL, &mbmi->skip, args->t_above, args->t_left, 1};
     vp10_encode_block_intra(plane, block, blk_row, blk_col,
                             plane_bsize, tx_size, &intra_arg);