shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix the transform type selection in 4x4 partition

This commit allows proper transform type (DCT/ADST) selection in
the settings of partition 4x4 level.

Change-Id: Iec6f922a46480d777e7ca9142a99e8c131f0077b

--- a/vp9/common/vp9_blockd.h

+++ b/vp9/common/vp9_blockd.h

@@ -625,13 +625,11 @@

   // is smaller than the prediction size

   TX_TYPE tx_type = DCT_DCT;

   const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;

-  const int wb = b_width_log2(sb_type), hb = b_height_log2(sb_type);

+  const int wb = b_width_log2(sb_type);

 #if !USE_ADST_FOR_SB

   if (sb_type > BLOCK_SIZE_MB16X16)

     return tx_type;

 #endif

-  if (ib >= (1 << (wb + hb)))  // no chroma adst

-    return tx_type;

   if (xd->lossless)

     return DCT_DCT;

   if (xd->mode_info_context->mbmi.mode == I4X4_PRED &&

--- a/vp9/encoder/vp9_rdopt.c

+++ b/vp9/encoder/vp9_rdopt.c

@@ -2331,8 +2331,7 @@

     *returnrate = rate_y + rate_uv - rate_y_tokenonly - rate_uv_tokenonly +

                   vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 1);

     *returndist = dist_y + (dist_uv >> 2);

-    memset(ctx->txfm_rd_diff, 0,

-           sizeof(x->sb32_context[xd->sb_index].txfm_rd_diff));

+    memset(ctx->txfm_rd_diff, 0, sizeof(ctx->txfm_rd_diff));

     xd->mode_info_context->mbmi.mode = mode;

     xd->mode_info_context->mbmi.txfm_size = txfm_size;

 #if CONFIG_AB4X4

-- 

⑨