vpx: [x86] vpx_hadamard_16x16_avx2() improvements
~10% performance gain. Fixed the cosmetics noted in the
previous commit.
Change-Id: Iddf475f34d0d0a3e356b2143682aeabac459ed13
diff --git a/vpx_dsp/x86/avg_intrin_avx2.c b/vpx_dsp/x86/avg_intrin_avx2.c
index 2ded29c..4192a45 100644
--- a/vpx_dsp/x86/avg_intrin_avx2.c
+++ b/vpx_dsp/x86/avg_intrin_avx2.c
@@ -106,47 +106,25 @@
hadamard_col8x2_avx2(src, 0);
hadamard_col8x2_avx2(src, 1);
- store_tran_low(_mm256_castsi256_si128(src[0]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[1]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[2]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[3]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[4]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[5]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[6]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[7]), coeff);
- coeff += 8;
+ // TODO(slavarnway): FIXME: For high bitdepths, it is unnecessary to
+ // mult/unpack/store here and load/pack the same memory in the next stage.
+ // Try using an intermediate buffer and store_tran_low() in the last stage.
+ store_tran_low(_mm256_permute2x128_si256(src[0], src[1], 0x20), coeff);
+ coeff += 16;
+ store_tran_low(_mm256_permute2x128_si256(src[2], src[3], 0x20), coeff);
+ coeff += 16;
+ store_tran_low(_mm256_permute2x128_si256(src[4], src[5], 0x20), coeff);
+ coeff += 16;
+ store_tran_low(_mm256_permute2x128_si256(src[6], src[7], 0x20), coeff);
+ coeff += 16;
- src[0] = _mm256_castsi128_si256(_mm256_extractf128_si256(src[0], 1));
- src[1] = _mm256_castsi128_si256(_mm256_extractf128_si256(src[1], 1));
- src[2] = _mm256_castsi128_si256(_mm256_extractf128_si256(src[2], 1));
- src[3] = _mm256_castsi128_si256(_mm256_extractf128_si256(src[3], 1));
- src[4] = _mm256_castsi128_si256(_mm256_extractf128_si256(src[4], 1));
- src[5] = _mm256_castsi128_si256(_mm256_extractf128_si256(src[5], 1));
- src[6] = _mm256_castsi128_si256(_mm256_extractf128_si256(src[6], 1));
- src[7] = _mm256_castsi128_si256(_mm256_extractf128_si256(src[7], 1));
-
- store_tran_low(_mm256_castsi256_si128(src[0]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[1]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[2]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[3]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[4]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[5]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[6]), coeff);
- coeff += 8;
- store_tran_low(_mm256_castsi256_si128(src[7]), coeff);
+ store_tran_low(_mm256_permute2x128_si256(src[0], src[1], 0x31), coeff);
+ coeff += 16;
+ store_tran_low(_mm256_permute2x128_si256(src[2], src[3], 0x31), coeff);
+ coeff += 16;
+ store_tran_low(_mm256_permute2x128_si256(src[4], src[5], 0x31), coeff);
+ coeff += 16;
+ store_tran_low(_mm256_permute2x128_si256(src[6], src[7], 0x31), coeff);
}
void vpx_hadamard_16x16_avx2(int16_t const *src_diff, int src_stride,
@@ -172,10 +150,10 @@
b2 = _mm256_srai_epi16(b2, 1);
b3 = _mm256_srai_epi16(b3, 1);
- store_tran_low_256(_mm256_add_epi16(b0, b2), coeff);
- store_tran_low_256(_mm256_add_epi16(b1, b3), coeff + 64);
- store_tran_low_256(_mm256_sub_epi16(b0, b2), coeff + 128);
- store_tran_low_256(_mm256_sub_epi16(b1, b3), coeff + 192);
+ store_tran_low(_mm256_add_epi16(b0, b2), coeff);
+ store_tran_low(_mm256_add_epi16(b1, b3), coeff + 64);
+ store_tran_low(_mm256_sub_epi16(b0, b2), coeff + 128);
+ store_tran_low(_mm256_sub_epi16(b1, b3), coeff + 192);
coeff += 16;
}
diff --git a/vpx_dsp/x86/bitdepth_conversion_avx2.h b/vpx_dsp/x86/bitdepth_conversion_avx2.h
index 7bbdac7..b8fd1cb 100644
--- a/vpx_dsp/x86/bitdepth_conversion_avx2.h
+++ b/vpx_dsp/x86/bitdepth_conversion_avx2.h
@@ -27,33 +27,17 @@
#endif
}
-// Store 8 16 bit values. If the destination is 32 bits then sign extend the
-// values by multiplying by 1.
-static INLINE void store_tran_low(__m128i a, tran_low_t *b) {
-#if CONFIG_VP9_HIGHBITDEPTH
- const __m128i one = _mm_set1_epi16(1);
- const __m128i a_hi = _mm_mulhi_epi16(a, one);
- const __m128i a_lo = _mm_mullo_epi16(a, one);
- const __m128i a_1 = _mm_unpacklo_epi16(a_lo, a_hi);
- const __m128i a_2 = _mm_unpackhi_epi16(a_lo, a_hi);
- _mm_store_si128((__m128i *)(b), a_1);
- _mm_store_si128((__m128i *)(b + 4), a_2);
-#else
- _mm_store_si128((__m128i *)(b), a);
-#endif
-}
-
-static INLINE void store_tran_low_256(__m256i a, tran_low_t *b) {
+static INLINE void store_tran_low(__m256i a, tran_low_t *b) {
#if CONFIG_VP9_HIGHBITDEPTH
const __m256i one = _mm256_set1_epi16(1);
const __m256i a_hi = _mm256_mulhi_epi16(a, one);
const __m256i a_lo = _mm256_mullo_epi16(a, one);
const __m256i a_1 = _mm256_unpacklo_epi16(a_lo, a_hi);
const __m256i a_2 = _mm256_unpackhi_epi16(a_lo, a_hi);
- _mm256_storeu_si256((__m256i *)(b), a_1);
+ _mm256_storeu_si256((__m256i *)b, a_1);
_mm256_storeu_si256((__m256i *)(b + 8), a_2);
#else
- _mm256_storeu_si256((__m256i *)(b), a);
+ _mm256_storeu_si256((__m256i *)b, a);
#endif
}
#endif // VPX_DSP_X86_BITDEPTH_CONVERSION_AVX2_H_
