commit 8468a5c1a0765ef3b6aad278f147b0e4142d913a
author Jingning Han <jingning@google.com> Wed May 15 10:19:40 2013 -0700
committer Jingning Han <jingning@google.com> Wed May 15 16:09:58 2013 -0700
tree 4bbb034e096dc5daf2d1815f52eec141d17f789e
parent cd16fe91606a698721dcc5802b29f6f859a629ff

Fix the transform type selection in 4x4 partition

This commit allows proper transform type (DCT/ADST) selection in
the settings of partition 4x4 level.

Change-Id: Iec6f922a46480d777e7ca9142a99e8c131f0077b

vp9/common/vp9_blockd.h

diff --git a/vp9/common/vp9_blockd.h b/vp9/common/vp9_blockd.h
index 07607d8..ad38730 100644
--- a/vp9/common/vp9_blockd.h
+++ b/vp9/common/vp9_blockd.h
@@ -625,13 +625,11 @@
   // is smaller than the prediction size
   TX_TYPE tx_type = DCT_DCT;
   const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
-  const int wb = b_width_log2(sb_type), hb = b_height_log2(sb_type);
+  const int wb = b_width_log2(sb_type);
 #if !USE_ADST_FOR_SB
   if (sb_type > BLOCK_SIZE_MB16X16)
     return tx_type;
 #endif
-  if (ib >= (1 << (wb + hb)))  // no chroma adst
-    return tx_type;
   if (xd->lossless)
     return DCT_DCT;
   if (xd->mode_info_context->mbmi.mode == I4X4_PRED &&

vp9/encoder/vp9_rdopt.c

diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 271a63f..5097664 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2331,8 +2331,7 @@
     *returnrate = rate_y + rate_uv - rate_y_tokenonly - rate_uv_tokenonly +
                   vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 1);
     *returndist = dist_y + (dist_uv >> 2);
-    memset(ctx->txfm_rd_diff, 0,
-           sizeof(x->sb32_context[xd->sb_index].txfm_rd_diff));
+    memset(ctx->txfm_rd_diff, 0, sizeof(ctx->txfm_rd_diff));
     xd->mode_info_context->mbmi.mode = mode;
     xd->mode_info_context->mbmi.txfm_size = txfm_size;
 #if CONFIG_AB4X4