Add macro to allow different tx sets for 16x16
This allows for the following options:
Set 0:
Inter: All 16 txfms
Intra: Discrete Trig transforms w/0 flip (4) + Identity (1) +
1D Hor/vert DCT (2)
Set 1:
Inter: Discrete Trig transforms w/ flip (9) + Identity (1) +
1D Hor/Ver DCT (2)
Intra: Discrete Trig transforms w/0 flip (4) + Identity (1)
Set 2:
Inter: Discrete Trig transforms w/ flip (9) + Identity (1)
Intra: Discrete Trig transforms w/0 flip (4) + Identity (1)
Results on lowres 40 frames with
disable-ext-partition disable-ext-partition-types
Set 0: 0.03%
Set 1: No change
Set 2: 0.06%
Change-Id: Iec57d8c8fcfa0891528de4ca88f54753dfcb5284
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 5e5fa29..180cbfa 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -765,25 +765,32 @@
// Number of transform types in each set type
static const int av1_num_ext_tx_set[EXT_TX_SET_TYPES] = {
- 1, 2, 5, 7, 10, 12, 16,
+ 1, 2, 5, 7, 7, 10, 12, 16, 16,
};
static const int av1_ext_tx_set_idx_to_type[2][AOMMAX(EXT_TX_SETS_INTRA,
EXT_TX_SETS_INTER)] = {
{
// Intra
- EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX,
+ EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT,
+#if EXT_TX_16X16_SET == 0
+ EXT_TX_SET_DTT4_IDTX_1DDCT_16X16,
+#else
+ EXT_TX_SET_DTT4_IDTX,
+#endif // EXT_TX_16X16_SET
},
{
// Inter
EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16,
-#if USE_1D_16X16
+#if EXT_TX_16X16_SET == 0
+ EXT_TX_SET_ALL16_16X16,
+#elif EXT_TX_16X16_SET == 1
EXT_TX_SET_DTT9_IDTX_1DDCT,
#else
EXT_TX_SET_DTT9_IDTX,
-#endif // USE_1D_16X16
+#endif // EXT_TX_16X16_SET
EXT_TX_SET_DCT_IDTX,
- }
+ },
};
static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = {
@@ -800,6 +807,9 @@
1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0,
},
{
+ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0,
+ },
+ {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
},
{
@@ -808,6 +818,9 @@
{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
},
+ {
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ },
};
static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
@@ -835,14 +848,22 @@
if (tx_size_sqr_up == TX_32X32)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
if (is_inter)
-#if USE_1D_16X16
- return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
- : EXT_TX_SET_ALL16);
+ return (tx_size_sqr == TX_16X16 ?
+#if EXT_TX_16X16_SET == 0
+ EXT_TX_SET_ALL16_16X16
+#elif EXT_TX_16X16_SET == 1
+ EXT_TX_SET_DTT9_IDTX_1DDCT
#else
- return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX : EXT_TX_SET_ALL16);
-#endif // USE_1D_16X16
+ EXT_TX_SET_DTT9_IDTX
+#endif // EXT_TX_16X16_SET
+ : EXT_TX_SET_ALL16);
else
- return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT4_IDTX
+ return (tx_size_sqr == TX_16X16 ?
+#if EXT_TX_16X16_SET == 0
+ EXT_TX_SET_DTT4_IDTX_1DDCT_16X16
+#else
+ EXT_TX_SET_DTT4_IDTX
+#endif // EXT_TX_16X16_SET
: EXT_TX_SET_DTT4_IDTX_1DDCT);
}
@@ -850,16 +871,24 @@
static const int ext_tx_set_index[2][EXT_TX_SET_TYPES] = {
{
// Intra
- 0, -1, 2, 1, -1, -1, -1,
- },
- {
- // Inter
- 0, 3, -1, -1,
-#if USE_1D_16X16
+ 0, -1,
+#if EXT_TX_16X16_SET == 0
-1, 2,
#else
2, -1,
-#endif // USE_1D_16X16
+#endif // EXT_TX_16X16_SET
+ 1, -1, -1, -1, -1,
+ },
+ {
+ // Inter
+ 0, 3, -1, -1, -1,
+#if EXT_TX_16X16_SET == 0
+ -1, -1, 2,
+#elif EXT_TX_16X16_SET == 1
+ -1, 2, -1,
+#else
+ 2, -1, -1,
+#endif // EXT_TX_16X16_SET
1,
},
};
diff --git a/av1/common/entropymode.c b/av1/common/entropymode.c
index 1aa0667..ef5f2d4 100644
--- a/av1/common/entropymode.c
+++ b/av1/common/entropymode.c
@@ -1935,6 +1935,70 @@
{ AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
},
},
+#if EXT_TX_16X16_SET == 0
+ {
+ {
+ { AOM_CDF7(1024, 28800, 29048, 29296, 30164, 31466) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1152, 25852, 26284, 26717, 28230, 30499) },
+ { AOM_CDF7(1024, 2016, 3938, 5860, 29404, 31086) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
+ { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ },
+ {
+ { AOM_CDF7(1024, 28800, 29048, 29296, 30164, 31466) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1152, 25852, 26284, 26717, 28230, 30499) },
+ { AOM_CDF7(1024, 2016, 3938, 5860, 29404, 31086) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
+ { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ },
+ {
+ { AOM_CDF7(1024, 28800, 29048, 29296, 30164, 31466) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1152, 25852, 26284, 26717, 28230, 30499) },
+ { AOM_CDF7(1024, 2016, 3938, 5860, 29404, 31086) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
+ { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ },
+ {
+ { AOM_CDF7(1024, 28800, 29048, 29296, 30164, 31466) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1152, 25852, 26284, 26717, 28230, 30499) },
+ { AOM_CDF7(1024, 2016, 3938, 5860, 29404, 31086) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
+ { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
+ },
+ },
+#else
{
{
{ AOM_CDF5(1024, 28800, 29792, 31280) },
@@ -1997,6 +2061,7 @@
{ AOM_CDF5(1280, 5216, 6938, 13396) },
},
},
+#endif // EXT_TX_16X16_SET == 0
};
static const aom_cdf_prob
default_inter_ext_tx_cdf[EXT_TX_SETS_INTER][EXT_TX_SIZES][CDF_SIZE(
@@ -2010,7 +2075,16 @@
21733, 24241, 26749, 28253, 29758, 31263) },
{ AOM_CDF16(1280, 1453, 1626, 2277, 2929, 3580, 4232, 16717, 19225,
21733, 24241, 26749, 28253, 29758, 31263) } },
-#if USE_1D_16X16
+#if EXT_TX_16X16_SET == 0
+ { { AOM_CDF16(1280, 1453, 1626, 2277, 2929, 3580, 4232, 16717, 19225,
+ 21733, 24241, 26749, 28253, 29758, 31263) },
+ { AOM_CDF16(1280, 1453, 1626, 2277, 2929, 3580, 4232, 16717, 19225,
+ 21733, 24241, 26749, 28253, 29758, 31263) },
+ { AOM_CDF16(1280, 1453, 1626, 2277, 2929, 3580, 4232, 16717, 19225,
+ 21733, 24241, 26749, 28253, 29758, 31263) },
+ { AOM_CDF16(1280, 1453, 1626, 2277, 2929, 3580, 4232, 16717, 19225,
+ 21733, 24241, 26749, 28253, 29758, 31263) } },
+#elif EXT_TX_16X16_SET == 1
{ { AOM_CDF12(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904, 28370,
29836, 31302) },
{ AOM_CDF12(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904, 28370,
@@ -2028,7 +2102,7 @@
28370) },
{ AOM_CDF10(1280, 3125, 4970, 17132, 19575, 22018, 24461, 26904,
28370) } },
-#endif // USE_1D_16X16
+#endif // EXT_TX_16X16_SET
{ { AOM_CDF2(1536) },
{ AOM_CDF2(1536) },
{ AOM_CDF2(1536) },
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 4df53d2..acb23c8 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -444,7 +444,19 @@
[CDF_SIZE(INTRA_MODES)];
#endif
-#define USE_1D_16X16 1
+// Decides what set to assign to 16x16 transforms.
+// Set 0:
+// Inter: All 16 txfms
+// Intra: Discrete Trig transforms w/0 flip (4) + Identity (1) +
+// 1D Hor/vert DCT (2)
+// Set 1:
+// Inter: Discrete Trig transforms w/ flip (9) + Identity (1) +
+// 1D Hor/Ver DCT (2)
+// Intra: Discrete Trig transforms w/0 flip (4) + Identity (1)
+// Set 2:
+// Inter: Discrete Trig transforms w/ flip (9) + Identity (1)
+// Intra: Discrete Trig transforms w/0 flip (4) + Identity (1)
+#define EXT_TX_16X16_SET 1
static const int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES] = {
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -459,6 +471,9 @@
1, 5, 6, 4, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0,
},
{
+ 1, 5, 6, 4, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0,
+ },
+ {
1, 2, 3, 6, 4, 5, 7, 8, 9, 0, 0, 0, 0, 0, 0, 0,
},
{
@@ -467,6 +482,9 @@
{
7, 8, 9, 12, 10, 11, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6,
},
+ {
+ 7, 8, 9, 12, 10, 11, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6,
+ },
};
static const int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES] = {
@@ -483,6 +501,9 @@
9, 0, 10, 11, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0,
},
{
+ 9, 0, 10, 11, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ },
+ {
9, 0, 1, 2, 4, 5, 3, 6, 7, 8, 0, 0, 0, 0, 0, 0,
},
{
@@ -491,6 +512,9 @@
{
9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 4, 5, 3, 6, 7, 8,
},
+ {
+ 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 4, 5, 3, 6, 7, 8,
+ },
};
extern const aom_tree_index
diff --git a/av1/common/enums.h b/av1/common/enums.h
index 7a82c5c..0829e64 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -356,12 +356,18 @@
// Discrete Trig transforms w/o flip (4) + Identity (1)
EXT_TX_SET_DTT4_IDTX,
// Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
+ // for 16x16 only
+ EXT_TX_SET_DTT4_IDTX_1DDCT_16X16,
+ // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
EXT_TX_SET_DTT4_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1)
EXT_TX_SET_DTT9_IDTX,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
EXT_TX_SET_DTT9_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
+ // for 16x16 only
+ EXT_TX_SET_ALL16_16X16,
+ // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
EXT_TX_SET_ALL16,
EXT_TX_SET_TYPES
} TxSetType;
