commit f4112212da523381777597ea3d920d38fce8d21f
author Debargha Mukherjee <debargha@google.com> Wed Aug 10 15:24:26 2016 -0700
committer Debargha Mukherjee <debargha@google.com> Thu Aug 11 10:01:54 2016 -0700
tree fecb06fa43d667d4a484b0f23fe4a7e5f256b4be
parent 6dde80181838b172f4d208b756661a75c09c89bc

A fix in optimize_b for new-quant

Change-Id: I5a7bd3c2d0c7f6cf714367674f1d75510659b54d

vp10/encoder/encodemb.c

diff --git a/vp10/encoder/encodemb.c b/vp10/encoder/encodemb.c
index aa8b402..46ee48d 100644
--- a/vp10/encoder/encodemb.c
+++ b/vp10/encoder/encodemb.c
@@ -89,12 +89,13 @@
       get_scan(tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
   const int16_t* const scan = so->scan;
   const int16_t* const nb = so->neighbors;
+  const int shift = get_tx_scale(xd, tx_type, tx_size);
 #if CONFIG_NEW_QUANT
   int dq = get_dq_profile_from_ctx(ctx);
   const dequant_val_type_nuq *dequant_val = pd->dequant_val_nuq[dq];
-#endif  // CONFIG_NEW_QUANT
-  const int shift = get_tx_scale(xd, tx_type, tx_size);
+#else
   const int dq_step[2] = { dequant_ptr[0] >> shift, dequant_ptr[1] >> shift };
+#endif  // CONFIG_NEW_QUANT
   int next = eob, sz = 0;
   const int64_t rdmult = (mb->rdmult * plane_rd_mult[ref][type]) >> 1;
   const int64_t rddiv = mb->rddiv;
@@ -296,6 +297,14 @@
       tokens[i][1].qc = x;
 
       if (x) {
+#if CONFIG_NEW_QUANT
+        tokens[i][1].dqc = vp10_dequant_abscoeff_nuq(
+            abs(x), dequant_ptr[rc != 0], dequant_val[band_translate[i]]);
+        tokens[i][1].dqc = shift ?
+            ROUND_POWER_OF_TWO(tokens[i][1].dqc, shift) : tokens[i][1].dqc;
+        if (sz)
+          tokens[i][1].dqc = -tokens[i][1].dqc;
+#else
         tran_low_t offset = dq_step[rc != 0];
         // The 32x32 transform coefficient uses half quantization step size.
         // Account for the rounding difference in the dequantized coefficeint
@@ -308,6 +317,7 @@
           tokens[i][1].dqc = dqcoeff[rc] - offset;
         else
           tokens[i][1].dqc = dqcoeff[rc] + offset;
+#endif  // CONFIG_NEW_QUANT
       } else {
         tokens[i][1].dqc = 0;
       }
@@ -365,13 +375,6 @@
     qcoeff[rc] = x;
     dqcoeff[rc] = tokens[i][best].dqc;
 
-#if CONFIG_NEW_QUANT
-    dqcoeff[rc] = vp10_dequant_abscoeff_nuq(abs(x), dequant_ptr[rc != 0],
-                                            dequant_val[band_translate[i]]);
-    if (shift) dqcoeff[rc] = ROUND_POWER_OF_TWO(dqcoeff[rc], shift);
-    if (x < 0) dqcoeff[rc] = -dqcoeff[rc];
-#endif  // CONFIG_NEW_QUANT
-
     next = tokens[i][best].next;
     best = best_index[i][best];
   }