commit 2bc3d47dd1fb65bd718116b4afb607c961a06872
author Hui Su <huisu@google.com> Tue Oct 16 09:50:13 2018 -0700
committer Hui Su <huisu@google.com> Tue Oct 16 10:52:08 2018 -0700
tree 7a1074cc03d2d6262209b3add7df6b5d9bed634f
parent b7e9a463f88982505f938557bc9dcea1178b39c2

Fix a bug in ml_prune_rect_partition()

The quantization step size should be scaled properly for high bit depth
settings.

This only affects speed 0.
Encoder speed change is almost neutral.
There is a small coding gain of 0.09%.

Change-Id: I96b2bae03a53ce8ccd6428e3a050cfe18e06a024

vp9/encoder/vp9_encodeframe.c

diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index ba1d71b..9864de0 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -3288,7 +3288,12 @@
   {
     const int64_t none_rdcost = pc_tree->none.rdcost;
     const VP9_COMMON *const cm = &cpi->common;
+#if CONFIG_VP9_HIGHBITDEPTH
+    const int dc_q =
+        vp9_dc_quant(cm->base_qindex, 0, cm->bit_depth) >> (x->e_mbd.bd - 8);
+#else
     const int dc_q = vp9_dc_quant(cm->base_qindex, 0, cm->bit_depth);
+#endif  // CONFIG_VP9_HIGHBITDEPTH
     int feature_index = 0;
     unsigned int block_var = 0;
     unsigned int sub_block_var[4] = { 0 };