commit 76550dfdc0a898cb9510977f79b43dbac1fd9de0
author paulwilkins <paulwilkins@google.com> Thu Feb 09 16:30:38 2017 +0000
committer paulwilkins <paulwilkins@google.com> Wed Feb 15 10:45:03 2017 +0000
tree 12c8b7075118d17f70684b543ce57909c70ff04b
parent 945ccfee59fe2d1bde5fc3f30845a6e0503bec59

Bug in scale_sse_threshold()

The function scale_sse_threshold() returns a threshold scaled
if necessary for use with 10 and 12 bit from an 8 bit baseline.

SSE error values would be expected to rise for the 10 and 12
bit cases where there are more bits of precision.

Hence the threshold used for the test should also be scaled up.

Change-Id: I4009c98b6eecd1bf64c3c38aaa56598e0136b03d

vp9/encoder/vp9_firstpass.c

diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c
index e087740..cdb957a 100644
--- a/vp9/encoder/vp9_firstpass.c
+++ b/vp9/encoder/vp9_firstpass.c
@@ -469,8 +469,8 @@
   if (cm->use_highbitdepth) {
     switch (cm->bit_depth) {
       case VPX_BITS_8: ret_val = thresh; break;
-      case VPX_BITS_10: ret_val = thresh >> 4; break;
-      case VPX_BITS_12: ret_val = thresh >> 8; break;
+      case VPX_BITS_10: ret_val = thresh << 4; break;
+      case VPX_BITS_12: ret_val = thresh << 8; break;
       default:
         assert(0 &&
                "cm->bit_depth should be VPX_BITS_8, "