Refactoring and clean ups for inv_txfm ranges
Change-Id: Ib37752952832cda36c323e8adcc564b1811b563a
diff --git a/av1/common/av1_inv_txfm1d_cfg.h b/av1/common/av1_inv_txfm1d_cfg.h
index 97beacf..3a2c6c2 100644
--- a/av1/common/av1_inv_txfm1d_cfg.h
+++ b/av1/common/av1_inv_txfm1d_cfg.h
@@ -15,12 +15,10 @@
// sum of fwd_shift_##
#if CONFIG_TX64X64
-static const int8_t fwd_shift_sum[TX_SIZES] = { 2, 1, 0, -2, -4 };
static const int8_t inv_start_range[TX_SIZES_ALL] = {
5, 6, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7
};
#else // CONFIG_TX64X64
-static const int8_t fwd_shift_sum[TX_SIZES] = { 2, 1, 0, -2 };
static const int8_t inv_start_range[TX_SIZES_ALL] = { 5, 6, 7, 7, 6, 6, 7,
7, 7, 7, 6, 6, 7, 7 };
#endif // CONFIG_TX64X64
@@ -30,12 +28,12 @@
static const int8_t inv_shift_4[2] = { 0, -4 };
// stage range
-static const int8_t inv_stage_range_col_dct_4[4] = { 3, 3, 3, 3 };
-static const int8_t inv_stage_range_row_dct_4[4] = { 3, 3, 3, 3 };
-static const int8_t inv_stage_range_col_adst_4[6] = { 3, 3, 3, 3, 2, 2 };
-static const int8_t inv_stage_range_row_adst_4[6] = { 3, 3, 3, 3, 3, 3 };
-static const int8_t inv_stage_range_row_idx_4[1] = { 3 };
-static const int8_t inv_stage_range_col_idx_4[1] = { 3 };
+static const int8_t inv_stage_range_col_dct_4[4] = { 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_dct_4[4] = { 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_4[6] = { 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_adst_4[6] = { 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_idx_4[1] = { 0 };
+static const int8_t inv_stage_range_col_idx_4[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_4[4] = { 13, 13, 13, 13 };
@@ -47,12 +45,12 @@
// shift
static const int8_t inv_shift_8[2] = { 0, -5 };
// stage range
-static const int8_t inv_stage_range_col_dct_8[6] = { 5, 5, 5, 5, 4, 4 };
-static const int8_t inv_stage_range_row_dct_8[6] = { 5, 5, 5, 5, 5, 5 };
-static const int8_t inv_stage_range_col_adst_8[8] = { 5, 5, 5, 5, 5, 5, 4, 4 };
-static const int8_t inv_stage_range_row_adst_8[8] = { 5, 5, 5, 5, 5, 5, 5, 5 };
-static const int8_t inv_stage_range_row_idx_8[1] = { 5 };
-static const int8_t inv_stage_range_col_idx_8[1] = { 5 };
+static const int8_t inv_stage_range_col_dct_8[6] = { 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_dct_8[6] = { 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_8[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_adst_8[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_idx_8[1] = { 0 };
+static const int8_t inv_stage_range_col_idx_8[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_8[6] = { 13, 13, 13, 13, 13, 13 };
@@ -67,10 +65,10 @@
// ---------------- 4x8 1D constants -----------------------
#define inv_shift_4x8 inv_shift_8
// stage range
-static const int8_t inv_stage_range_col_dct_4x8[6] =
- ARRAYOFFSET6(-2, 5, 5, 5, 5, 5, 5);
-static const int8_t inv_stage_range_col_adst_4x8[8] =
- ARRAYOFFSET8(-2, 5, 5, 5, 5, 5, 5, 5, 5);
+static const int8_t inv_stage_range_col_dct_4x8[6] = { 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_4x8[8] = {
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
// cos bit
static const int8_t inv_cos_bit_col_dct_4x8[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_4x8[8] = { 13, 13, 13, 13,
@@ -79,10 +77,8 @@
// ---------------- 8x4 1D constants -----------------------
#define inv_shift_8x4 inv_shift_8
// stage range
-static const int8_t inv_stage_range_col_dct_8x4[4] =
- ARRAYOFFSET4(2, 3, 3, 3, 3);
-static const int8_t inv_stage_range_col_adst_8x4[6] =
- ARRAYOFFSET6(2, 3, 3, 3, 3, 3, 3);
+static const int8_t inv_stage_range_col_dct_8x4[4] = { 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_8x4[6] = { 0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_8x4[4] = { 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_8x4[6] = { 13, 13, 13, 13, 13, 13 };
@@ -92,14 +88,14 @@
static const int8_t inv_shift_16[2] = { -1, -5 };
// stage range
-static const int8_t inv_stage_range_col_dct_16[8] = { 7, 7, 7, 7, 7, 7, 6, 6 };
-static const int8_t inv_stage_range_row_dct_16[8] = { 7, 7, 7, 7, 7, 7, 7, 7 };
-static const int8_t inv_stage_range_col_adst_16[10] = { 7, 7, 7, 7, 7,
- 7, 7, 7, 6, 6 };
-static const int8_t inv_stage_range_row_adst_16[10] = { 7, 7, 7, 7, 7,
- 7, 7, 7, 7, 7 };
-static const int8_t inv_stage_range_row_idx_16[1] = { 7 };
-static const int8_t inv_stage_range_col_idx_16[1] = { 7 };
+static const int8_t inv_stage_range_col_dct_16[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_dct_16[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_16[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_adst_16[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_idx_16[1] = { 0 };
+static const int8_t inv_stage_range_col_idx_16[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_16[8] = {
@@ -116,14 +112,15 @@
// ---------------- 8x16 1D constants -----------------------
#define inv_shift_8x16 inv_shift_16
// stage range
-static const int8_t inv_stage_range_row_dct_8x16[6] = { 5, 5, 5, 5, 5, 5 };
+static const int8_t inv_stage_range_row_dct_8x16[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_adst_8x16[8] = {
- 5, 5, 5, 5, 5, 5, 5, 5
+ 0, 0, 0, 0, 0, 0, 0, 0
};
-static const int8_t inv_stage_range_col_dct_8x16[8] =
- ARRAYOFFSET8(-2, 7, 7, 7, 7, 7, 7, 7, 7);
-static const int8_t inv_stage_range_col_adst_8x16[10] =
- ARRAYOFFSET10(-2, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7);
+static const int8_t inv_stage_range_col_dct_8x16[8] = {
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
+static const int8_t inv_stage_range_col_adst_8x16[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_row_dct_8x16[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t inv_cos_bit_row_adst_8x16[8] = { 12, 12, 12, 12,
@@ -136,10 +133,10 @@
// ---------------- 16x8 1D constants -----------------------
#define inv_shift_16x8 inv_shift_16
// stage range
-static const int8_t inv_stage_range_col_dct_16x8[6] =
- ARRAYOFFSET6(2, 5, 5, 5, 5, 5, 5);
-static const int8_t inv_stage_range_col_adst_16x8[8] =
- ARRAYOFFSET8(2, 5, 5, 5, 5, 5, 5, 5, 5);
+static const int8_t inv_stage_range_col_dct_16x8[6] = { 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_16x8[8] = {
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
// cos bit
static const int8_t inv_cos_bit_col_dct_16x8[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_16x8[8] = { 13, 13, 13, 13,
@@ -150,16 +147,16 @@
static const int8_t inv_shift_32[2] = { -1, -5 };
// stage range
-static const int8_t inv_stage_range_col_dct_32[10] = { 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9 };
-static const int8_t inv_stage_range_row_dct_32[10] = { 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9 };
-static const int8_t inv_stage_range_col_adst_32[12] = { 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9 };
-static const int8_t inv_stage_range_row_adst_32[12] = { 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9 };
-static const int8_t inv_stage_range_row_idx_32[1] = { 9 };
-static const int8_t inv_stage_range_col_idx_32[1] = { 9 };
+static const int8_t inv_stage_range_col_dct_32[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_dct_32[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_32[12] = { 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_adst_32[12] = { 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_idx_32[1] = { 0 };
+static const int8_t inv_stage_range_col_idx_32[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_32[10] = { 13, 13, 13, 13, 13,
@@ -174,10 +171,10 @@
// ---------------- 16x32 1D constants -----------------------
#define inv_shift_16x32 inv_shift_32
// stage range
-static const int8_t inv_stage_range_col_dct_16x32[10] =
- ARRAYOFFSET10(-2, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
-static const int8_t inv_stage_range_col_adst_16x32[12] =
- ARRAYOFFSET12(-2, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
+static const int8_t inv_stage_range_col_dct_16x32[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_16x32[12] = { 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_16x32[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
@@ -187,10 +184,11 @@
// ---------------- 32x16 1D constants -----------------------
#define inv_shift_32x16 inv_shift_32
// stage range
-static const int8_t inv_stage_range_col_dct_32x16[8] =
- ARRAYOFFSET8(2, 7, 7, 7, 7, 7, 7, 7, 7);
-static const int8_t inv_stage_range_col_adst_32x16[10] =
- ARRAYOFFSET10(2, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7);
+static const int8_t inv_stage_range_col_dct_32x16[8] = {
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
+static const int8_t inv_stage_range_col_adst_32x16[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_32x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
@@ -202,13 +200,13 @@
static const int8_t inv_shift_64[2] = { -1, -5 };
// stage range
-static const int8_t inv_stage_range_col_dct_64[12] = { 11, 11, 11, 11, 11, 11,
- 11, 11, 11, 11, 11, 11 };
-static const int8_t inv_stage_range_row_dct_64[12] = { 11, 11, 11, 11, 11, 11,
- 11, 11, 11, 11, 11, 11 };
+static const int8_t inv_stage_range_col_dct_64[12] = { 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_row_dct_64[12] = { 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0 };
-static const int8_t inv_stage_range_row_idx_64[1] = { 11 };
-static const int8_t inv_stage_range_col_idx_64[1] = { 11 };
+static const int8_t inv_stage_range_row_idx_64[1] = { 0 };
+static const int8_t inv_stage_range_col_idx_64[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_64[12] = { 13, 13, 13, 13, 13, 13,
@@ -219,8 +217,8 @@
// ---------------- 32x64 1D constants -----------------------
#define inv_shift_32x64 inv_shift_64
// stage range
-static const int8_t inv_stage_range_col_dct_32x64[12] =
- ARRAYOFFSET12(-2, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11);
+static const int8_t inv_stage_range_col_dct_32x64[12] = { 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_32x64[12] = { 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13 };
@@ -228,8 +226,8 @@
// ---------------- 64x32 1D constants -----------------------
#define inv_shift_64x32 inv_shift_64
// stage range
-static const int8_t inv_stage_range_col_dct_64x32[10] =
- ARRAYOFFSET10(2, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
+static const int8_t inv_stage_range_col_dct_64x32[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_64x32[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
@@ -237,10 +235,11 @@
// ---------------- 4x16 1D constants -----------------------
#define inv_shift_4x16 inv_shift_16
// stage range
-static const int8_t inv_stage_range_col_dct_4x16[8] =
- ARRAYOFFSET8(-4, 7, 7, 7, 7, 7, 7, 7, 7);
-static const int8_t inv_stage_range_col_adst_4x16[10] =
- ARRAYOFFSET10(-4, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7);
+static const int8_t inv_stage_range_col_dct_4x16[8] = {
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
+static const int8_t inv_stage_range_col_adst_4x16[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_4x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
@@ -250,10 +249,8 @@
// ---------------- 16x4 1D constants -----------------------
#define inv_shift_16x4 inv_shift_16
// stage range
-static const int8_t inv_stage_range_col_dct_16x4[4] =
- ARRAYOFFSET4(4, 3, 3, 3, 3);
-static const int8_t inv_stage_range_col_adst_16x4[6] =
- ARRAYOFFSET6(4, 3, 3, 3, 3, 3, 3);
+static const int8_t inv_stage_range_col_dct_16x4[4] = { 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_16x4[6] = { 0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_16x4[4] = { 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_16x4[6] = { 13, 13, 13, 13, 13, 13 };
@@ -261,14 +258,14 @@
// ---------------- 8x32 1D constants -----------------------
#define inv_shift_8x32 inv_shift_32
// stage range
-static const int8_t inv_stage_range_row_dct_8x32[6] = { 5, 5, 5, 5, 5, 5 };
+static const int8_t inv_stage_range_row_dct_8x32[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_adst_8x32[8] = {
- 5, 5, 5, 5, 5, 5, 5, 5
+ 0, 0, 0, 0, 0, 0, 0, 0
};
-static const int8_t inv_stage_range_col_dct_8x32[10] =
- ARRAYOFFSET10(-4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
-static const int8_t inv_stage_range_col_adst_8x32[12] =
- ARRAYOFFSET12(-4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
+static const int8_t inv_stage_range_col_dct_8x32[10] = { 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_8x32[12] = { 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_row_dct_8x32[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t inv_cos_bit_row_adst_8x32[8] = { 12, 12, 12, 12,
@@ -281,10 +278,10 @@
// ---------------- 32x8 1D constants -----------------------
#define inv_shift_32x8 inv_shift_32
// stage range
-static const int8_t inv_stage_range_col_dct_32x8[6] =
- ARRAYOFFSET6(4, 5, 5, 5, 5, 5, 5);
-static const int8_t inv_stage_range_col_adst_32x8[8] =
- ARRAYOFFSET8(4, 5, 5, 5, 5, 5, 5, 5, 5);
+static const int8_t inv_stage_range_col_dct_32x8[6] = { 0, 0, 0, 0, 0, 0 };
+static const int8_t inv_stage_range_col_adst_32x8[8] = {
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
// cos bit
static const int8_t inv_cos_bit_col_dct_32x8[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_32x8[8] = { 13, 13, 13, 13,
@@ -293,8 +290,8 @@
// ---------------- 16x64 1D constants -----------------------
#define inv_shift_16x64 inv_shift_64
// stage range
-static const int8_t inv_stage_range_col_dct_16x64[12] =
- ARRAYOFFSET12(-4, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11);
+static const int8_t inv_stage_range_col_dct_16x64[12] = { 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_16x64[12] = { 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13 };
@@ -302,8 +299,9 @@
// ---------------- 64x16 1D constants -----------------------
#define inv_shift_64x16 inv_shift_64
// stage range
-static const int8_t inv_stage_range_col_dct_64x16[8] =
- ARRAYOFFSET8(4, 7, 7, 7, 7, 7, 7, 7, 7);
+static const int8_t inv_stage_range_col_dct_64x16[8] = {
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
// cos bit
static const int8_t inv_cos_bit_col_dct_64x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
diff --git a/av1/common/av1_inv_txfm2d.c b/av1/common/av1_inv_txfm2d.c
index 3c83769..9714572 100644
--- a/av1/common/av1_inv_txfm2d.c
+++ b/av1/common/av1_inv_txfm2d.c
@@ -256,7 +256,6 @@
// rectangular, the number of columns will be the same as the
// txfm_size stored in the row cfg struct. It will make no difference
// for square transforms.
- // const int fwd_shift = fwd_shift_sum[txsize_sqr_up_map[tx_size]];
const int fwd_shift = inv_start_range[tx_size];
const int txfm_size_col = cfg->row_cfg->txfm_size;
const int txfm_size_row = cfg->col_cfg->txfm_size;
@@ -279,14 +278,12 @@
}
// i < MAX_TXFM_STAGE_NUM will mute above array bounds warning
for (int i = 0; i < cfg->row_cfg->stage_num && i < MAX_TXFM_STAGE_NUM; ++i) {
- stage_range_row[i] = cfg->row_cfg->stage_range[i] + fwd_shift + bd + 1 -
- cfg->row_cfg->stage_range[0];
+ stage_range_row[i] = cfg->row_cfg->stage_range[i] + fwd_shift + bd + 1;
}
// i < MAX_TXFM_STAGE_NUM will mute above array bounds warning
for (int i = 0; i < cfg->col_cfg->stage_num && i < MAX_TXFM_STAGE_NUM; ++i) {
stage_range_col[i] = cfg->col_cfg->stage_range[i] + fwd_shift + shift[0] +
- bd + 1 - cfg->col_cfg->stage_range[0] +
- rect_type2_shift;
+ bd + 1 + rect_type2_shift;
}
}
