commit 16f77e31bcad8873bcf5d37e5df77e80107fbb8e
author Jingning Han <jingning@google.com> Wed Feb 07 15:50:20 2018 -0800
committer Jingning Han <jingning@google.com> Thu Feb 08 02:57:21 2018 +0000
tree 2529fb7eb56ae34aac6a81c4fb6ba87449fe2efa
parent a6f987bafe3cf9b25cae58d1b63e0d0db4b1ea46

Make txb hash map support high bit-depth

When the data path is in high bit-depth, the transform block hash
map may send different residual pixel blocks into same hash index.
Fix this issue by scaling the DC level used to offset the residual
pixels according to the bit depth.

BUG=aomedia:1339

Change-Id: Ic4310f547f260321d3a656774430c8fe859c55d8

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index ac0a0c9..c84785f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4681,6 +4681,7 @@
   int cur_tx_depth = 0;
   uint8_t parent_idx_buf[MAX_MIB_SIZE * MAX_MIB_SIZE] = { 0 };
   uint8_t child_idx_buf[MAX_MIB_SIZE * MAX_MIB_SIZE] = { 0 };
+  const int hash_dc_level = 1 << x->e_mbd.bd;
 
   TX_SIZE cur_tx_size = max_txsize_rect_lookup[1][bsize];
   while (cur_tx_depth <= MAX_VARTX_DEPTH) {
@@ -4705,7 +4706,7 @@
           for (int i = 0; i < cur_tx_bh; i++) {
             const int16_t *cur_diff_row = diff + (row + i) * diff_stride + col;
             for (int j = 0; j < cur_tx_bw; j++) {
-              hash = hash ^ clip_pixel(cur_diff_row[j] + 128);
+              hash = hash ^ (cur_diff_row[j] + hash_dc_level);
               hash = (uint32_t)((int64_t)hash * 16777619);
             }
           }