| // Copyright 2019 The libgav1 Authors |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "src/dsp/intrapred.h" |
| #include "src/utils/cpu.h" |
| |
| #if LIBGAV1_ENABLE_SSE4_1 |
| |
| #include <xmmintrin.h> |
| |
| #include <algorithm> |
| #include <cassert> |
| #include <cstddef> |
| #include <cstdint> |
| #include <cstring> // memcpy |
| |
| #include "src/dsp/constants.h" |
| #include "src/dsp/dsp.h" |
| #include "src/dsp/x86/common_sse4.h" |
| #include "src/dsp/x86/transpose_sse4.h" |
| #include "src/utils/common.h" |
| |
| namespace libgav1 { |
| namespace dsp { |
| namespace { |
| |
| //------------------------------------------------------------------------------ |
| // Utility Functions |
| |
| // This is a fast way to divide by a number of the form 2^n + 2^k, n > k. |
| // Divide by 2^k by right shifting by k, leaving the denominator 2^m + 1. In the |
| // block size cases, n - k is 1 or 2 (block is proportional to 1x2 or 1x4), so |
| // we use a multiplier that reflects division by 2+1=3 or 4+1=5 in the high |
| // bits. |
| constexpr int kThreeInverse = 0x5556; |
| constexpr int kFiveInverse = 0x3334; |
| template <int shiftk, int multiplier> |
| inline __m128i DivideByMultiplyShift_U32(const __m128i dividend) { |
| const __m128i interm = _mm_srli_epi32(dividend, shiftk); |
| return _mm_mulhi_epi16(interm, _mm_cvtsi32_si128(multiplier)); |
| } |
| |
| // This shuffle mask selects 32-bit blocks in the order 0, 1, 0, 1, which |
| // duplicates the first 8 bytes of a 128-bit vector into the second 8 bytes. |
| constexpr int kDuplicateFirstHalf = 0x44; |
| |
| //------------------------------------------------------------------------------ |
| // DcPredFuncs_SSE4_1 |
| |
| using DcSumFunc = __m128i (*)(const void* ref); |
| using DcStoreFunc = void (*)(void* dest, ptrdiff_t stride, const __m128i dc); |
| using WriteDuplicateFunc = void (*)(void* dest, ptrdiff_t stride, |
| const __m128i column); |
| // For copying an entire column across a block. |
| using ColumnStoreFunc = void (*)(void* dest, ptrdiff_t stride, |
| const void* column); |
| |
| // DC intra-predictors for non-square blocks. |
| template <int width_log2, int height_log2, DcSumFunc top_sumfn, |
| DcSumFunc left_sumfn, DcStoreFunc storefn, int shiftk, int dc_mult> |
| struct DcPredFuncs_SSE4_1 { |
| DcPredFuncs_SSE4_1() = delete; |
| |
| static void DcTop(void* dest, ptrdiff_t stride, const void* top_row, |
| const void* left_column); |
| static void DcLeft(void* dest, ptrdiff_t stride, const void* top_row, |
| const void* left_column); |
| static void Dc(void* dest, ptrdiff_t stride, const void* top_row, |
| const void* left_column); |
| }; |
| |
| // Directional intra-predictors for square blocks. |
| template <ColumnStoreFunc col_storefn> |
| struct DirectionalPredFuncs_SSE4_1 { |
| DirectionalPredFuncs_SSE4_1() = delete; |
| |
| static void Vertical(void* dest, ptrdiff_t stride, const void* top_row, |
| const void* left_column); |
| static void Horizontal(void* dest, ptrdiff_t stride, const void* top_row, |
| const void* left_column); |
| }; |
| |
| template <int width_log2, int height_log2, DcSumFunc top_sumfn, |
| DcSumFunc left_sumfn, DcStoreFunc storefn, int shiftk, int dc_mult> |
| void DcPredFuncs_SSE4_1<width_log2, height_log2, top_sumfn, left_sumfn, storefn, |
| shiftk, dc_mult>::DcTop(void* const dest, |
| ptrdiff_t stride, |
| const void* const top_row, |
| const void* /*left_column*/) { |
| const __m128i rounder = _mm_set1_epi32(1 << (width_log2 - 1)); |
| const __m128i sum = top_sumfn(top_row); |
| const __m128i dc = _mm_srli_epi32(_mm_add_epi32(sum, rounder), width_log2); |
| storefn(dest, stride, dc); |
| } |
| |
| template <int width_log2, int height_log2, DcSumFunc top_sumfn, |
| DcSumFunc left_sumfn, DcStoreFunc storefn, int shiftk, int dc_mult> |
| void DcPredFuncs_SSE4_1<width_log2, height_log2, top_sumfn, left_sumfn, storefn, |
| shiftk, |
| dc_mult>::DcLeft(void* const dest, ptrdiff_t stride, |
| const void* /*top_row*/, |
| const void* const left_column) { |
| const __m128i rounder = _mm_set1_epi32(1 << (height_log2 - 1)); |
| const __m128i sum = left_sumfn(left_column); |
| const __m128i dc = _mm_srli_epi32(_mm_add_epi32(sum, rounder), height_log2); |
| storefn(dest, stride, dc); |
| } |
| |
| template <int width_log2, int height_log2, DcSumFunc top_sumfn, |
| DcSumFunc left_sumfn, DcStoreFunc storefn, int shiftk, int dc_mult> |
| void DcPredFuncs_SSE4_1<width_log2, height_log2, top_sumfn, left_sumfn, storefn, |
| shiftk, dc_mult>::Dc(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i rounder = |
| _mm_set1_epi32((1 << (width_log2 - 1)) + (1 << (height_log2 - 1))); |
| const __m128i sum_top = top_sumfn(top_row); |
| const __m128i sum_left = left_sumfn(left_column); |
| const __m128i sum = _mm_add_epi32(sum_top, sum_left); |
| if (width_log2 == height_log2) { |
| const __m128i dc = |
| _mm_srli_epi32(_mm_add_epi32(sum, rounder), width_log2 + 1); |
| storefn(dest, stride, dc); |
| } else { |
| const __m128i dc = |
| DivideByMultiplyShift_U32<shiftk, dc_mult>(_mm_add_epi32(sum, rounder)); |
| storefn(dest, stride, dc); |
| } |
| } |
| |
| //------------------------------------------------------------------------------ |
| // DcPredFuncs_SSE4_1 directional predictors |
| |
| template <ColumnStoreFunc col_storefn> |
| void DirectionalPredFuncs_SSE4_1<col_storefn>::Horizontal( |
| void* const dest, ptrdiff_t stride, const void* /*top_row*/, |
| const void* const left_column) { |
| col_storefn(dest, stride, left_column); |
| } |
| |
| } // namespace |
| |
| //------------------------------------------------------------------------------ |
| namespace low_bitdepth { |
| namespace { |
| |
| // |ref| points to 4 bytes containing 4 packed ints. |
| inline __m128i DcSum4_SSE4_1(const void* const ref) { |
| const __m128i vals = Load4(ref); |
| const __m128i zero = _mm_setzero_si128(); |
| return _mm_sad_epu8(vals, zero); |
| } |
| |
| inline __m128i DcSum8_SSE4_1(const void* const ref) { |
| const __m128i vals = LoadLo8(ref); |
| const __m128i zero = _mm_setzero_si128(); |
| return _mm_sad_epu8(vals, zero); |
| } |
| |
| inline __m128i DcSum16_SSE4_1(const void* const ref) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i vals = LoadUnaligned16(ref); |
| const __m128i partial_sum = _mm_sad_epu8(vals, zero); |
| return _mm_add_epi16(partial_sum, _mm_srli_si128(partial_sum, 8)); |
| } |
| |
| inline __m128i DcSum32_SSE4_1(const void* const ref) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i vals1 = LoadUnaligned16(ref); |
| const __m128i vals2 = LoadUnaligned16(static_cast<const uint8_t*>(ref) + 16); |
| const __m128i partial_sum1 = _mm_sad_epu8(vals1, zero); |
| const __m128i partial_sum2 = _mm_sad_epu8(vals2, zero); |
| const __m128i partial_sum = _mm_add_epi16(partial_sum1, partial_sum2); |
| return _mm_add_epi16(partial_sum, _mm_srli_si128(partial_sum, 8)); |
| } |
| |
| inline __m128i DcSum64_SSE4_1(const void* const ref) { |
| const auto* const ref_ptr = static_cast<const uint8_t*>(ref); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i vals1 = LoadUnaligned16(ref_ptr); |
| const __m128i vals2 = LoadUnaligned16(ref_ptr + 16); |
| const __m128i vals3 = LoadUnaligned16(ref_ptr + 32); |
| const __m128i vals4 = LoadUnaligned16(ref_ptr + 48); |
| const __m128i partial_sum1 = _mm_sad_epu8(vals1, zero); |
| const __m128i partial_sum2 = _mm_sad_epu8(vals2, zero); |
| __m128i partial_sum = _mm_add_epi16(partial_sum1, partial_sum2); |
| const __m128i partial_sum3 = _mm_sad_epu8(vals3, zero); |
| partial_sum = _mm_add_epi16(partial_sum, partial_sum3); |
| const __m128i partial_sum4 = _mm_sad_epu8(vals4, zero); |
| partial_sum = _mm_add_epi16(partial_sum, partial_sum4); |
| return _mm_add_epi16(partial_sum, _mm_srli_si128(partial_sum, 8)); |
| } |
| |
| template <int height> |
| inline void DcStore4xH_SSE4_1(void* const dest, ptrdiff_t stride, |
| const __m128i dc) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i dc_dup = _mm_shuffle_epi8(dc, zero); |
| int y = height - 1; |
| auto* dst = static_cast<uint8_t*>(dest); |
| do { |
| Store4(dst, dc_dup); |
| dst += stride; |
| } while (--y != 0); |
| Store4(dst, dc_dup); |
| } |
| |
| template <int height> |
| inline void DcStore8xH_SSE4_1(void* const dest, ptrdiff_t stride, |
| const __m128i dc) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i dc_dup = _mm_shuffle_epi8(dc, zero); |
| int y = height - 1; |
| auto* dst = static_cast<uint8_t*>(dest); |
| do { |
| StoreLo8(dst, dc_dup); |
| dst += stride; |
| } while (--y != 0); |
| StoreLo8(dst, dc_dup); |
| } |
| |
| template <int height> |
| inline void DcStore16xH_SSE4_1(void* const dest, ptrdiff_t stride, |
| const __m128i dc) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i dc_dup = _mm_shuffle_epi8(dc, zero); |
| int y = height - 1; |
| auto* dst = static_cast<uint8_t*>(dest); |
| do { |
| StoreUnaligned16(dst, dc_dup); |
| dst += stride; |
| } while (--y != 0); |
| StoreUnaligned16(dst, dc_dup); |
| } |
| |
| template <int height> |
| inline void DcStore32xH_SSE4_1(void* const dest, ptrdiff_t stride, |
| const __m128i dc) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i dc_dup = _mm_shuffle_epi8(dc, zero); |
| int y = height - 1; |
| auto* dst = static_cast<uint8_t*>(dest); |
| do { |
| StoreUnaligned16(dst, dc_dup); |
| StoreUnaligned16(dst + 16, dc_dup); |
| dst += stride; |
| } while (--y != 0); |
| StoreUnaligned16(dst, dc_dup); |
| StoreUnaligned16(dst + 16, dc_dup); |
| } |
| |
| template <int height> |
| inline void DcStore64xH_SSE4_1(void* const dest, ptrdiff_t stride, |
| const __m128i dc) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i dc_dup = _mm_shuffle_epi8(dc, zero); |
| int y = height - 1; |
| auto* dst = static_cast<uint8_t*>(dest); |
| do { |
| StoreUnaligned16(dst, dc_dup); |
| StoreUnaligned16(dst + 16, dc_dup); |
| StoreUnaligned16(dst + 32, dc_dup); |
| StoreUnaligned16(dst + 48, dc_dup); |
| dst += stride; |
| } while (--y != 0); |
| StoreUnaligned16(dst, dc_dup); |
| StoreUnaligned16(dst + 16, dc_dup); |
| StoreUnaligned16(dst + 32, dc_dup); |
| StoreUnaligned16(dst + 48, dc_dup); |
| } |
| |
| // WriteDuplicateN assumes dup has 4 sets of 4 identical bytes that are meant to |
| // be copied for width N into dest. |
| inline void WriteDuplicate4x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| auto* dst = static_cast<uint8_t*>(dest); |
| Store4(dst, dup32); |
| dst += stride; |
| const int row1 = _mm_extract_epi32(dup32, 1); |
| memcpy(dst, &row1, 4); |
| dst += stride; |
| const int row2 = _mm_extract_epi32(dup32, 2); |
| memcpy(dst, &row2, 4); |
| dst += stride; |
| const int row3 = _mm_extract_epi32(dup32, 3); |
| memcpy(dst, &row3, 4); |
| } |
| |
| inline void WriteDuplicate8x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| auto* dst = static_cast<uint8_t*>(dest); |
| _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), dup64_lo); |
| dst += stride; |
| _mm_storeh_pi(reinterpret_cast<__m64*>(dst), _mm_castsi128_ps(dup64_lo)); |
| dst += stride; |
| _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), dup64_hi); |
| dst += stride; |
| _mm_storeh_pi(reinterpret_cast<__m64*>(dst), _mm_castsi128_ps(dup64_hi)); |
| } |
| |
| inline void WriteDuplicate16x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i dup128_0 = _mm_unpacklo_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_0); |
| dst += stride; |
| const __m128i dup128_1 = _mm_unpackhi_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_1); |
| dst += stride; |
| const __m128i dup128_2 = _mm_unpacklo_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_2); |
| dst += stride; |
| const __m128i dup128_3 = _mm_unpackhi_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_3); |
| } |
| |
| inline void WriteDuplicate32x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i dup128_0 = _mm_unpacklo_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_0); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_0); |
| dst += stride; |
| const __m128i dup128_1 = _mm_unpackhi_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_1); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_1); |
| dst += stride; |
| const __m128i dup128_2 = _mm_unpacklo_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_2); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_2); |
| dst += stride; |
| const __m128i dup128_3 = _mm_unpackhi_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_3); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_3); |
| } |
| |
| inline void WriteDuplicate64x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i dup128_0 = _mm_unpacklo_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_0); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_0); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), dup128_0); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), dup128_0); |
| dst += stride; |
| const __m128i dup128_1 = _mm_unpackhi_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_1); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_1); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), dup128_1); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), dup128_1); |
| dst += stride; |
| const __m128i dup128_2 = _mm_unpacklo_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_2); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_2); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), dup128_2); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), dup128_2); |
| dst += stride; |
| const __m128i dup128_3 = _mm_unpackhi_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_3); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_3); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), dup128_3); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), dup128_3); |
| } |
| |
| // ColStoreN<height> copies each of the |height| values in |column| across its |
| // corresponding in dest. |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore4_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const __m128i col_data = Load4(column); |
| const __m128i col_dup16 = _mm_unpacklo_epi8(col_data, col_data); |
| const __m128i col_dup32 = _mm_unpacklo_epi16(col_dup16, col_dup16); |
| writefn(dest, stride, col_dup32); |
| } |
| |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore8_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const ptrdiff_t stride4 = stride << 2; |
| const __m128i col_data = LoadLo8(column); |
| const __m128i col_dup16 = _mm_unpacklo_epi8(col_data, col_data); |
| const __m128i col_dup32_lo = _mm_unpacklo_epi16(col_dup16, col_dup16); |
| auto* dst = static_cast<uint8_t*>(dest); |
| writefn(dst, stride, col_dup32_lo); |
| dst += stride4; |
| const __m128i col_dup32_hi = _mm_unpackhi_epi16(col_dup16, col_dup16); |
| writefn(dst, stride, col_dup32_hi); |
| } |
| |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore16_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const ptrdiff_t stride4 = stride << 2; |
| const __m128i col_data = _mm_loadu_si128(static_cast<const __m128i*>(column)); |
| const __m128i col_dup16_lo = _mm_unpacklo_epi8(col_data, col_data); |
| const __m128i col_dup16_hi = _mm_unpackhi_epi8(col_data, col_data); |
| const __m128i col_dup32_lolo = _mm_unpacklo_epi16(col_dup16_lo, col_dup16_lo); |
| auto* dst = static_cast<uint8_t*>(dest); |
| writefn(dst, stride, col_dup32_lolo); |
| dst += stride4; |
| const __m128i col_dup32_lohi = _mm_unpackhi_epi16(col_dup16_lo, col_dup16_lo); |
| writefn(dst, stride, col_dup32_lohi); |
| dst += stride4; |
| const __m128i col_dup32_hilo = _mm_unpacklo_epi16(col_dup16_hi, col_dup16_hi); |
| writefn(dst, stride, col_dup32_hilo); |
| dst += stride4; |
| const __m128i col_dup32_hihi = _mm_unpackhi_epi16(col_dup16_hi, col_dup16_hi); |
| writefn(dst, stride, col_dup32_hihi); |
| } |
| |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore32_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const ptrdiff_t stride4 = stride << 2; |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y = 0; y < 32; y += 16) { |
| const __m128i col_data = |
| LoadUnaligned16(static_cast<const uint8_t*>(column) + y); |
| const __m128i col_dup16_lo = _mm_unpacklo_epi8(col_data, col_data); |
| const __m128i col_dup16_hi = _mm_unpackhi_epi8(col_data, col_data); |
| const __m128i col_dup32_lolo = |
| _mm_unpacklo_epi16(col_dup16_lo, col_dup16_lo); |
| writefn(dst, stride, col_dup32_lolo); |
| dst += stride4; |
| const __m128i col_dup32_lohi = |
| _mm_unpackhi_epi16(col_dup16_lo, col_dup16_lo); |
| writefn(dst, stride, col_dup32_lohi); |
| dst += stride4; |
| const __m128i col_dup32_hilo = |
| _mm_unpacklo_epi16(col_dup16_hi, col_dup16_hi); |
| writefn(dst, stride, col_dup32_hilo); |
| dst += stride4; |
| const __m128i col_dup32_hihi = |
| _mm_unpackhi_epi16(col_dup16_hi, col_dup16_hi); |
| writefn(dst, stride, col_dup32_hihi); |
| dst += stride4; |
| } |
| } |
| |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore64_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const ptrdiff_t stride4 = stride << 2; |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y = 0; y < 64; y += 16) { |
| const __m128i col_data = |
| LoadUnaligned16(static_cast<const uint8_t*>(column) + y); |
| const __m128i col_dup16_lo = _mm_unpacklo_epi8(col_data, col_data); |
| const __m128i col_dup16_hi = _mm_unpackhi_epi8(col_data, col_data); |
| const __m128i col_dup32_lolo = |
| _mm_unpacklo_epi16(col_dup16_lo, col_dup16_lo); |
| writefn(dst, stride, col_dup32_lolo); |
| dst += stride4; |
| const __m128i col_dup32_lohi = |
| _mm_unpackhi_epi16(col_dup16_lo, col_dup16_lo); |
| writefn(dst, stride, col_dup32_lohi); |
| dst += stride4; |
| const __m128i col_dup32_hilo = |
| _mm_unpacklo_epi16(col_dup16_hi, col_dup16_hi); |
| writefn(dst, stride, col_dup32_hilo); |
| dst += stride4; |
| const __m128i col_dup32_hihi = |
| _mm_unpackhi_epi16(col_dup16_hi, col_dup16_hi); |
| writefn(dst, stride, col_dup32_hihi); |
| dst += stride4; |
| } |
| } |
| |
| struct DcDefs { |
| DcDefs() = delete; |
| |
| using _4x4 = DcPredFuncs_SSE4_1<2, 2, DcSum4_SSE4_1, DcSum4_SSE4_1, |
| DcStore4xH_SSE4_1<4>, 0, 0>; |
| // shiftk is the smaller of width_log2 and height_log2. |
| // dc_mult corresponds to the ratio of the smaller block size to the larger. |
| using _4x8 = DcPredFuncs_SSE4_1<2, 3, DcSum4_SSE4_1, DcSum8_SSE4_1, |
| DcStore4xH_SSE4_1<8>, 2, kThreeInverse>; |
| using _4x16 = DcPredFuncs_SSE4_1<2, 4, DcSum4_SSE4_1, DcSum16_SSE4_1, |
| DcStore4xH_SSE4_1<16>, 2, kFiveInverse>; |
| |
| using _8x4 = DcPredFuncs_SSE4_1<3, 2, DcSum8_SSE4_1, DcSum4_SSE4_1, |
| DcStore8xH_SSE4_1<4>, 2, kThreeInverse>; |
| using _8x8 = DcPredFuncs_SSE4_1<3, 3, DcSum8_SSE4_1, DcSum8_SSE4_1, |
| DcStore8xH_SSE4_1<8>, 0, 0>; |
| using _8x16 = DcPredFuncs_SSE4_1<3, 4, DcSum8_SSE4_1, DcSum16_SSE4_1, |
| DcStore8xH_SSE4_1<16>, 3, kThreeInverse>; |
| using _8x32 = DcPredFuncs_SSE4_1<3, 5, DcSum8_SSE4_1, DcSum32_SSE4_1, |
| DcStore8xH_SSE4_1<32>, 3, kFiveInverse>; |
| |
| using _16x4 = DcPredFuncs_SSE4_1<4, 2, DcSum16_SSE4_1, DcSum4_SSE4_1, |
| DcStore16xH_SSE4_1<4>, 2, kFiveInverse>; |
| using _16x8 = DcPredFuncs_SSE4_1<4, 3, DcSum16_SSE4_1, DcSum8_SSE4_1, |
| DcStore16xH_SSE4_1<8>, 3, kThreeInverse>; |
| using _16x16 = DcPredFuncs_SSE4_1<4, 4, DcSum16_SSE4_1, DcSum16_SSE4_1, |
| DcStore16xH_SSE4_1<16>, 0, 0>; |
| using _16x32 = DcPredFuncs_SSE4_1<4, 5, DcSum16_SSE4_1, DcSum32_SSE4_1, |
| DcStore16xH_SSE4_1<32>, 4, kThreeInverse>; |
| using _16x64 = DcPredFuncs_SSE4_1<4, 6, DcSum16_SSE4_1, DcSum64_SSE4_1, |
| DcStore16xH_SSE4_1<64>, 4, kFiveInverse>; |
| |
| using _32x8 = DcPredFuncs_SSE4_1<5, 3, DcSum32_SSE4_1, DcSum8_SSE4_1, |
| DcStore32xH_SSE4_1<8>, 3, kFiveInverse>; |
| using _32x16 = DcPredFuncs_SSE4_1<5, 4, DcSum32_SSE4_1, DcSum16_SSE4_1, |
| DcStore32xH_SSE4_1<16>, 4, kThreeInverse>; |
| using _32x32 = DcPredFuncs_SSE4_1<5, 5, DcSum32_SSE4_1, DcSum32_SSE4_1, |
| DcStore32xH_SSE4_1<32>, 0, 0>; |
| using _32x64 = DcPredFuncs_SSE4_1<5, 6, DcSum32_SSE4_1, DcSum64_SSE4_1, |
| DcStore32xH_SSE4_1<64>, 5, kThreeInverse>; |
| |
| using _64x16 = DcPredFuncs_SSE4_1<6, 4, DcSum64_SSE4_1, DcSum16_SSE4_1, |
| DcStore64xH_SSE4_1<16>, 4, kFiveInverse>; |
| using _64x32 = DcPredFuncs_SSE4_1<6, 5, DcSum64_SSE4_1, DcSum32_SSE4_1, |
| DcStore64xH_SSE4_1<32>, 5, kThreeInverse>; |
| using _64x64 = DcPredFuncs_SSE4_1<6, 6, DcSum64_SSE4_1, DcSum64_SSE4_1, |
| DcStore64xH_SSE4_1<64>, 0, 0>; |
| }; |
| |
| struct DirDefs { |
| DirDefs() = delete; |
| |
| using _4x4 = DirectionalPredFuncs_SSE4_1<ColStore4_SSE4_1<WriteDuplicate4x4>>; |
| using _4x8 = DirectionalPredFuncs_SSE4_1<ColStore8_SSE4_1<WriteDuplicate4x4>>; |
| using _4x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate4x4>>; |
| using _8x4 = DirectionalPredFuncs_SSE4_1<ColStore4_SSE4_1<WriteDuplicate8x4>>; |
| using _8x8 = DirectionalPredFuncs_SSE4_1<ColStore8_SSE4_1<WriteDuplicate8x4>>; |
| using _8x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate8x4>>; |
| using _8x32 = |
| DirectionalPredFuncs_SSE4_1<ColStore32_SSE4_1<WriteDuplicate8x4>>; |
| using _16x4 = |
| DirectionalPredFuncs_SSE4_1<ColStore4_SSE4_1<WriteDuplicate16x4>>; |
| using _16x8 = |
| DirectionalPredFuncs_SSE4_1<ColStore8_SSE4_1<WriteDuplicate16x4>>; |
| using _16x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate16x4>>; |
| using _16x32 = |
| DirectionalPredFuncs_SSE4_1<ColStore32_SSE4_1<WriteDuplicate16x4>>; |
| using _16x64 = |
| DirectionalPredFuncs_SSE4_1<ColStore64_SSE4_1<WriteDuplicate16x4>>; |
| using _32x8 = |
| DirectionalPredFuncs_SSE4_1<ColStore8_SSE4_1<WriteDuplicate32x4>>; |
| using _32x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate32x4>>; |
| using _32x32 = |
| DirectionalPredFuncs_SSE4_1<ColStore32_SSE4_1<WriteDuplicate32x4>>; |
| using _32x64 = |
| DirectionalPredFuncs_SSE4_1<ColStore64_SSE4_1<WriteDuplicate32x4>>; |
| using _64x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate64x4>>; |
| using _64x32 = |
| DirectionalPredFuncs_SSE4_1<ColStore32_SSE4_1<WriteDuplicate64x4>>; |
| using _64x64 = |
| DirectionalPredFuncs_SSE4_1<ColStore64_SSE4_1<WriteDuplicate64x4>>; |
| }; |
| |
| template <int y_mask> |
| inline void WritePaethLine4(uint8_t* dst, const __m128i& top, |
| const __m128i& left, const __m128i& top_lefts, |
| const __m128i& top_dists, const __m128i& left_dists, |
| const __m128i& top_left_diffs) { |
| const __m128i top_dists_y = _mm_shuffle_epi32(top_dists, y_mask); |
| |
| const __m128i lefts_y = _mm_shuffle_epi32(left, y_mask); |
| const __m128i top_left_dists = |
| _mm_abs_epi32(_mm_add_epi32(lefts_y, top_left_diffs)); |
| |
| // Section 7.11.2.2 specifies the logic and terms here. The less-or-equal |
| // operation is unavailable, so the logic for selecting top, left, or |
| // top_left is inverted. |
| __m128i not_select_left = _mm_cmpgt_epi32(left_dists, top_left_dists); |
| not_select_left = |
| _mm_or_si128(not_select_left, _mm_cmpgt_epi32(left_dists, top_dists_y)); |
| const __m128i not_select_top = _mm_cmpgt_epi32(top_dists_y, top_left_dists); |
| |
| const __m128i left_out = _mm_andnot_si128(not_select_left, lefts_y); |
| |
| const __m128i top_left_out = _mm_and_si128(not_select_top, top_lefts); |
| __m128i top_or_top_left_out = _mm_andnot_si128(not_select_top, top); |
| top_or_top_left_out = _mm_or_si128(top_or_top_left_out, top_left_out); |
| top_or_top_left_out = _mm_and_si128(not_select_left, top_or_top_left_out); |
| |
| // The sequence of 32-bit packed operations was found (see CL via blame) to |
| // outperform 16-bit operations, despite the availability of the packus |
| // function, when tested on a Xeon E7 v3. |
| const __m128i cvtepi32_epi8 = _mm_set1_epi32(0x0C080400); |
| const __m128i pred = _mm_shuffle_epi8( |
| _mm_or_si128(left_out, top_or_top_left_out), cvtepi32_epi8); |
| Store4(dst, pred); |
| } |
| |
| // top_left_diffs is the only variable whose ints may exceed 8 bits. Otherwise |
| // we would be able to do all of these operations as epi8 for a 16-pixel version |
| // of this function. Still, since lefts_y is just a vector of duplicates, it |
| // could pay off to accommodate top_left_dists for cmpgt, and repack into epi8 |
| // for the blends. |
| template <int y_mask> |
| inline void WritePaethLine8(uint8_t* dst, const __m128i& top, |
| const __m128i& left, const __m128i& top_lefts, |
| const __m128i& top_dists, const __m128i& left_dists, |
| const __m128i& top_left_diffs) { |
| const __m128i select_y = _mm_set1_epi32(y_mask); |
| const __m128i top_dists_y = _mm_shuffle_epi8(top_dists, select_y); |
| |
| const __m128i lefts_y = _mm_shuffle_epi8(left, select_y); |
| const __m128i top_left_dists = |
| _mm_abs_epi16(_mm_add_epi16(lefts_y, top_left_diffs)); |
| |
| // Section 7.11.2.2 specifies the logic and terms here. The less-or-equal |
| // operation is unavailable, so the logic for selecting top, left, or |
| // top_left is inverted. |
| __m128i not_select_left = _mm_cmpgt_epi16(left_dists, top_left_dists); |
| not_select_left = |
| _mm_or_si128(not_select_left, _mm_cmpgt_epi16(left_dists, top_dists_y)); |
| const __m128i not_select_top = _mm_cmpgt_epi16(top_dists_y, top_left_dists); |
| |
| const __m128i left_out = _mm_andnot_si128(not_select_left, lefts_y); |
| |
| const __m128i top_left_out = _mm_and_si128(not_select_top, top_lefts); |
| __m128i top_or_top_left_out = _mm_andnot_si128(not_select_top, top); |
| top_or_top_left_out = _mm_or_si128(top_or_top_left_out, top_left_out); |
| top_or_top_left_out = _mm_and_si128(not_select_left, top_or_top_left_out); |
| |
| const __m128i pred = _mm_packus_epi16( |
| _mm_or_si128(left_out, top_or_top_left_out), /* unused */ left_out); |
| _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), pred); |
| } |
| |
| // |top| is an epi8 of length 16 |
| // |left| is epi8 of unknown length, as y_mask specifies access |
| // |top_lefts| is an epi8 of 16 duplicates |
| // |top_dists| is an epi8 of unknown length, as y_mask specifies access |
| // |left_dists| is an epi8 of length 16 |
| // |left_dists_lo| is an epi16 of length 8 |
| // |left_dists_hi| is an epi16 of length 8 |
| // |top_left_diffs_lo| is an epi16 of length 8 |
| // |top_left_diffs_hi| is an epi16 of length 8 |
| // The latter two vectors are epi16 because their values may reach -510. |
| // |left_dists| is provided alongside its spread out version because it doesn't |
| // change between calls and interacts with both kinds of packing. |
| template <int y_mask> |
| inline void WritePaethLine16(uint8_t* dst, const __m128i& top, |
| const __m128i& left, const __m128i& top_lefts, |
| const __m128i& top_dists, |
| const __m128i& left_dists, |
| const __m128i& left_dists_lo, |
| const __m128i& left_dists_hi, |
| const __m128i& top_left_diffs_lo, |
| const __m128i& top_left_diffs_hi) { |
| const __m128i select_y = _mm_set1_epi32(y_mask); |
| const __m128i top_dists_y8 = _mm_shuffle_epi8(top_dists, select_y); |
| const __m128i top_dists_y16 = _mm_cvtepu8_epi16(top_dists_y8); |
| const __m128i lefts_y8 = _mm_shuffle_epi8(left, select_y); |
| const __m128i lefts_y16 = _mm_cvtepu8_epi16(lefts_y8); |
| |
| const __m128i top_left_dists_lo = |
| _mm_abs_epi16(_mm_add_epi16(lefts_y16, top_left_diffs_lo)); |
| const __m128i top_left_dists_hi = |
| _mm_abs_epi16(_mm_add_epi16(lefts_y16, top_left_diffs_hi)); |
| |
| const __m128i left_gt_top_left_lo = _mm_packs_epi16( |
| _mm_cmpgt_epi16(left_dists_lo, top_left_dists_lo), left_dists_lo); |
| const __m128i left_gt_top_left_hi = |
| _mm_packs_epi16(_mm_cmpgt_epi16(left_dists_hi, top_left_dists_hi), |
| /* unused second arg for pack */ left_dists_hi); |
| const __m128i left_gt_top_left = _mm_alignr_epi8( |
| left_gt_top_left_hi, _mm_slli_si128(left_gt_top_left_lo, 8), 8); |
| |
| const __m128i not_select_top_lo = |
| _mm_packs_epi16(_mm_cmpgt_epi16(top_dists_y16, top_left_dists_lo), |
| /* unused second arg for pack */ top_dists_y16); |
| const __m128i not_select_top_hi = |
| _mm_packs_epi16(_mm_cmpgt_epi16(top_dists_y16, top_left_dists_hi), |
| /* unused second arg for pack */ top_dists_y16); |
| const __m128i not_select_top = _mm_alignr_epi8( |
| not_select_top_hi, _mm_slli_si128(not_select_top_lo, 8), 8); |
| |
| const __m128i left_leq_top = |
| _mm_cmpeq_epi8(left_dists, _mm_min_epu8(top_dists_y8, left_dists)); |
| const __m128i select_left = _mm_andnot_si128(left_gt_top_left, left_leq_top); |
| |
| // Section 7.11.2.2 specifies the logic and terms here. The less-or-equal |
| // operation is unavailable, so the logic for selecting top, left, or |
| // top_left is inverted. |
| const __m128i left_out = _mm_and_si128(select_left, lefts_y8); |
| |
| const __m128i top_left_out = _mm_and_si128(not_select_top, top_lefts); |
| __m128i top_or_top_left_out = _mm_andnot_si128(not_select_top, top); |
| top_or_top_left_out = _mm_or_si128(top_or_top_left_out, top_left_out); |
| top_or_top_left_out = _mm_andnot_si128(select_left, top_or_top_left_out); |
| const __m128i pred = _mm_or_si128(left_out, top_or_top_left_out); |
| |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), pred); |
| } |
| |
| void Paeth4x4_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, const void* const left_column) { |
| const __m128i left = _mm_cvtepu8_epi32(Load4(left_column)); |
| const __m128i top = _mm_cvtepu8_epi32(Load4(top_row)); |
| |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_lefts = _mm_set1_epi32(top_ptr[-1]); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top[-1], |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| const __m128i left_dists = _mm_abs_epi32(_mm_sub_epi32(top, top_lefts)); |
| const __m128i top_dists = _mm_abs_epi32(_mm_sub_epi32(left, top_lefts)); |
| |
| const __m128i top_left_x2 = _mm_add_epi32(top_lefts, top_lefts); |
| const __m128i top_left_diff = _mm_sub_epi32(top, top_left_x2); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine4<0>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0x55>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xAA>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xFF>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| } |
| |
| void Paeth4x8_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, const void* const left_column) { |
| const __m128i left = LoadLo8(left_column); |
| const __m128i left_lo = _mm_cvtepu8_epi32(left); |
| const __m128i left_hi = _mm_cvtepu8_epi32(_mm_srli_si128(left, 4)); |
| |
| const __m128i top = _mm_cvtepu8_epi32(Load4(top_row)); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_lefts = _mm_set1_epi32(top_ptr[-1]); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top[-1], |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| const __m128i left_dists = _mm_abs_epi32(_mm_sub_epi32(top, top_lefts)); |
| const __m128i top_dists_lo = _mm_abs_epi32(_mm_sub_epi32(left_lo, top_lefts)); |
| const __m128i top_dists_hi = _mm_abs_epi32(_mm_sub_epi32(left_hi, top_lefts)); |
| |
| const __m128i top_left_x2 = _mm_add_epi32(top_lefts, top_lefts); |
| const __m128i top_left_diff = _mm_sub_epi32(top, top_left_x2); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine4<0>(dst, top, left_lo, top_lefts, top_dists_lo, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0x55>(dst, top, left_lo, top_lefts, top_dists_lo, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xAA>(dst, top, left_lo, top_lefts, top_dists_lo, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xFF>(dst, top, left_lo, top_lefts, top_dists_lo, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0>(dst, top, left_hi, top_lefts, top_dists_hi, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0x55>(dst, top, left_hi, top_lefts, top_dists_hi, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xAA>(dst, top, left_hi, top_lefts, top_dists_hi, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xFF>(dst, top, left_hi, top_lefts, top_dists_hi, left_dists, |
| top_left_diff); |
| } |
| |
| void Paeth4x16_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i left = LoadUnaligned16(left_column); |
| const __m128i left_0 = _mm_cvtepu8_epi32(left); |
| const __m128i left_1 = _mm_cvtepu8_epi32(_mm_srli_si128(left, 4)); |
| const __m128i left_2 = _mm_cvtepu8_epi32(_mm_srli_si128(left, 8)); |
| const __m128i left_3 = _mm_cvtepu8_epi32(_mm_srli_si128(left, 12)); |
| |
| const __m128i top = _mm_cvtepu8_epi32(Load4(top_row)); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_lefts = _mm_set1_epi32(top_ptr[-1]); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top[-1], |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| const __m128i left_dists = _mm_abs_epi32(_mm_sub_epi32(top, top_lefts)); |
| const __m128i top_dists_0 = _mm_abs_epi32(_mm_sub_epi32(left_0, top_lefts)); |
| const __m128i top_dists_1 = _mm_abs_epi32(_mm_sub_epi32(left_1, top_lefts)); |
| const __m128i top_dists_2 = _mm_abs_epi32(_mm_sub_epi32(left_2, top_lefts)); |
| const __m128i top_dists_3 = _mm_abs_epi32(_mm_sub_epi32(left_3, top_lefts)); |
| |
| const __m128i top_left_x2 = _mm_add_epi32(top_lefts, top_lefts); |
| const __m128i top_left_diff = _mm_sub_epi32(top, top_left_x2); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine4<0>(dst, top, left_0, top_lefts, top_dists_0, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0x55>(dst, top, left_0, top_lefts, top_dists_0, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xAA>(dst, top, left_0, top_lefts, top_dists_0, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xFF>(dst, top, left_0, top_lefts, top_dists_0, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0>(dst, top, left_1, top_lefts, top_dists_1, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0x55>(dst, top, left_1, top_lefts, top_dists_1, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xAA>(dst, top, left_1, top_lefts, top_dists_1, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xFF>(dst, top, left_1, top_lefts, top_dists_1, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0>(dst, top, left_2, top_lefts, top_dists_2, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0x55>(dst, top, left_2, top_lefts, top_dists_2, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xAA>(dst, top, left_2, top_lefts, top_dists_2, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xFF>(dst, top, left_2, top_lefts, top_dists_2, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0>(dst, top, left_3, top_lefts, top_dists_3, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0x55>(dst, top, left_3, top_lefts, top_dists_3, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xAA>(dst, top, left_3, top_lefts, top_dists_3, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine4<0xFF>(dst, top, left_3, top_lefts, top_dists_3, left_dists, |
| top_left_diff); |
| } |
| |
| void Paeth8x4_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, const void* const left_column) { |
| const __m128i left = _mm_cvtepu8_epi16(Load4(left_column)); |
| const __m128i top = _mm_cvtepu8_epi16(LoadLo8(top_row)); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_lefts = _mm_set1_epi16(top_ptr[-1]); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top[-1], |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| const __m128i left_dists = _mm_abs_epi16(_mm_sub_epi16(top, top_lefts)); |
| const __m128i top_dists = _mm_abs_epi16(_mm_sub_epi16(left, top_lefts)); |
| |
| const __m128i top_left_x2 = _mm_add_epi16(top_lefts, top_lefts); |
| const __m128i top_left_diff = _mm_sub_epi16(top, top_left_x2); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine8<0x01000100>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x03020302>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x05040504>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x07060706>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| } |
| |
| void Paeth8x8_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, const void* const left_column) { |
| const __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| const __m128i top = _mm_cvtepu8_epi16(LoadLo8(top_row)); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_lefts = _mm_set1_epi16(top_ptr[-1]); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top[-1], |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| const __m128i left_dists = _mm_abs_epi16(_mm_sub_epi16(top, top_lefts)); |
| const __m128i top_dists = _mm_abs_epi16(_mm_sub_epi16(left, top_lefts)); |
| |
| const __m128i top_left_x2 = _mm_add_epi16(top_lefts, top_lefts); |
| const __m128i top_left_diff = _mm_sub_epi16(top, top_left_x2); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine8<0x01000100>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x03020302>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x05040504>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x07060706>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x09080908>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0B0A0B0A>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0D0C0D0C>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0F0E0F0E>(dst, top, left, top_lefts, top_dists, left_dists, |
| top_left_diff); |
| } |
| |
| void Paeth8x16_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i left = LoadUnaligned16(left_column); |
| const __m128i left_lo = _mm_cvtepu8_epi16(left); |
| const __m128i left_hi = _mm_cvtepu8_epi16(_mm_srli_si128(left, 8)); |
| const __m128i top = _mm_cvtepu8_epi16(LoadLo8(top_row)); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_lefts = _mm_set1_epi16(top_ptr[-1]); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top[-1], |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| const __m128i left_dists = _mm_abs_epi16(_mm_sub_epi16(top, top_lefts)); |
| const __m128i top_dists_lo = _mm_abs_epi16(_mm_sub_epi16(left_lo, top_lefts)); |
| const __m128i top_dists_hi = _mm_abs_epi16(_mm_sub_epi16(left_hi, top_lefts)); |
| |
| const __m128i top_left_x2 = _mm_add_epi16(top_lefts, top_lefts); |
| const __m128i top_left_diff = _mm_sub_epi16(top, top_left_x2); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine8<0x01000100>(dst, top, left_lo, top_lefts, top_dists_lo, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x03020302>(dst, top, left_lo, top_lefts, top_dists_lo, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x05040504>(dst, top, left_lo, top_lefts, top_dists_lo, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x07060706>(dst, top, left_lo, top_lefts, top_dists_lo, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x09080908>(dst, top, left_lo, top_lefts, top_dists_lo, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0B0A0B0A>(dst, top, left_lo, top_lefts, top_dists_lo, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0D0C0D0C>(dst, top, left_lo, top_lefts, top_dists_lo, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0F0E0F0E>(dst, top, left_lo, top_lefts, top_dists_lo, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x01000100>(dst, top, left_hi, top_lefts, top_dists_hi, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x03020302>(dst, top, left_hi, top_lefts, top_dists_hi, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x05040504>(dst, top, left_hi, top_lefts, top_dists_hi, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x07060706>(dst, top, left_hi, top_lefts, top_dists_hi, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x09080908>(dst, top, left_hi, top_lefts, top_dists_hi, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0B0A0B0A>(dst, top, left_hi, top_lefts, top_dists_hi, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0D0C0D0C>(dst, top, left_hi, top_lefts, top_dists_hi, |
| left_dists, top_left_diff); |
| dst += stride; |
| WritePaethLine8<0x0F0E0F0E>(dst, top, left_hi, top_lefts, top_dists_hi, |
| left_dists, top_left_diff); |
| } |
| |
| void Paeth8x32_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* const dst = static_cast<uint8_t*>(dest); |
| Paeth8x16_SSE4_1(dst, stride, top_row, left_column); |
| Paeth8x16_SSE4_1(dst + (stride << 4), stride, top_row, left_ptr + 16); |
| } |
| |
| void Paeth16x4_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i left = Load4(left_column); |
| const __m128i top = LoadUnaligned16(top_row); |
| const __m128i top_lo = _mm_cvtepu8_epi16(top); |
| const __m128i top_hi = _mm_cvtepu8_epi16(_mm_srli_si128(top, 8)); |
| |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_lefts16 = _mm_set1_epi16(top_ptr[-1]); |
| const __m128i top_lefts8 = _mm_set1_epi8(static_cast<int8_t>(top_ptr[-1])); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top[-1], |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| |
| const __m128i left_dists = _mm_or_si128(_mm_subs_epu8(top, top_lefts8), |
| _mm_subs_epu8(top_lefts8, top)); |
| const __m128i left_dists_lo = _mm_cvtepu8_epi16(left_dists); |
| const __m128i left_dists_hi = |
| _mm_cvtepu8_epi16(_mm_srli_si128(left_dists, 8)); |
| const __m128i top_dists = _mm_or_si128(_mm_subs_epu8(left, top_lefts8), |
| _mm_subs_epu8(top_lefts8, left)); |
| |
| const __m128i top_left_x2 = _mm_add_epi16(top_lefts16, top_lefts16); |
| const __m128i top_left_diff_lo = _mm_sub_epi16(top_lo, top_left_x2); |
| const __m128i top_left_diff_hi = _mm_sub_epi16(top_hi, top_left_x2); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine16<0>(dst, top, left, top_lefts8, top_dists, left_dists, |
| left_dists_lo, left_dists_hi, top_left_diff_lo, |
| top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x01010101>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x02020202>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x03030303>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| } |
| |
| // Inlined for calling with offsets in larger transform sizes, mainly to |
| // preserve top_left. |
| inline void WritePaeth16x8(void* const dest, ptrdiff_t stride, |
| const uint8_t top_left, const __m128i top, |
| const __m128i left) { |
| const __m128i top_lo = _mm_cvtepu8_epi16(top); |
| const __m128i top_hi = _mm_cvtepu8_epi16(_mm_srli_si128(top, 8)); |
| |
| const __m128i top_lefts16 = _mm_set1_epi16(top_left); |
| const __m128i top_lefts8 = _mm_set1_epi8(static_cast<int8_t>(top_left)); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top_left, |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| |
| const __m128i left_dists = _mm_or_si128(_mm_subs_epu8(top, top_lefts8), |
| _mm_subs_epu8(top_lefts8, top)); |
| const __m128i left_dists_lo = _mm_cvtepu8_epi16(left_dists); |
| const __m128i left_dists_hi = |
| _mm_cvtepu8_epi16(_mm_srli_si128(left_dists, 8)); |
| const __m128i top_dists = _mm_or_si128(_mm_subs_epu8(left, top_lefts8), |
| _mm_subs_epu8(top_lefts8, left)); |
| |
| const __m128i top_left_x2 = _mm_add_epi16(top_lefts16, top_lefts16); |
| const __m128i top_left_diff_lo = _mm_sub_epi16(top_lo, top_left_x2); |
| const __m128i top_left_diff_hi = _mm_sub_epi16(top_hi, top_left_x2); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine16<0>(dst, top, left, top_lefts8, top_dists, left_dists, |
| left_dists_lo, left_dists_hi, top_left_diff_lo, |
| top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x01010101>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x02020202>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x03030303>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x04040404>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x05050505>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x06060606>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x07070707>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| } |
| |
| void Paeth16x8_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i top = LoadUnaligned16(top_row); |
| const __m128i left = LoadLo8(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| WritePaeth16x8(static_cast<uint8_t*>(dest), stride, top_ptr[-1], top, left); |
| } |
| |
| void WritePaeth16x16(void* const dest, ptrdiff_t stride, const uint8_t top_left, |
| const __m128i top, const __m128i left) { |
| const __m128i top_lo = _mm_cvtepu8_epi16(top); |
| const __m128i top_hi = _mm_cvtepu8_epi16(_mm_srli_si128(top, 8)); |
| |
| const __m128i top_lefts16 = _mm_set1_epi16(top_left); |
| const __m128i top_lefts8 = _mm_set1_epi8(static_cast<int8_t>(top_left)); |
| |
| // Given that the spec defines "base" as top[x] + left[y] - top[-1], |
| // pLeft = abs(base - left[y]) = abs(top[x] - top[-1]) |
| // pTop = abs(base - top[x]) = abs(left[y] - top[-1]) |
| |
| const __m128i left_dists = _mm_or_si128(_mm_subs_epu8(top, top_lefts8), |
| _mm_subs_epu8(top_lefts8, top)); |
| const __m128i left_dists_lo = _mm_cvtepu8_epi16(left_dists); |
| const __m128i left_dists_hi = |
| _mm_cvtepu8_epi16(_mm_srli_si128(left_dists, 8)); |
| const __m128i top_dists = _mm_or_si128(_mm_subs_epu8(left, top_lefts8), |
| _mm_subs_epu8(top_lefts8, left)); |
| |
| const __m128i top_left_x2 = _mm_add_epi16(top_lefts16, top_lefts16); |
| const __m128i top_left_diff_lo = _mm_sub_epi16(top_lo, top_left_x2); |
| const __m128i top_left_diff_hi = _mm_sub_epi16(top_hi, top_left_x2); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaethLine16<0>(dst, top, left, top_lefts8, top_dists, left_dists, |
| left_dists_lo, left_dists_hi, top_left_diff_lo, |
| top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x01010101>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x02020202>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x03030303>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x04040404>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x05050505>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x06060606>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x07070707>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x08080808>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x09090909>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x0A0A0A0A>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x0B0B0B0B>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x0C0C0C0C>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x0D0D0D0D>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x0E0E0E0E>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| dst += stride; |
| WritePaethLine16<0x0F0F0F0F>(dst, top, left, top_lefts8, top_dists, |
| left_dists, left_dists_lo, left_dists_hi, |
| top_left_diff_lo, top_left_diff_hi); |
| } |
| |
| void Paeth16x16_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i left = LoadUnaligned16(left_column); |
| const __m128i top = LoadUnaligned16(top_row); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| WritePaeth16x16(static_cast<uint8_t*>(dest), stride, top_ptr[-1], top, left); |
| } |
| |
| void Paeth16x32_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i left_0 = LoadUnaligned16(left_column); |
| const __m128i top = LoadUnaligned16(top_row); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* const dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x16(dst, stride, top_left, top, left_0); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i left_1 = LoadUnaligned16(left_ptr + 16); |
| WritePaeth16x16(dst + (stride << 4), stride, top_left, top, left_1); |
| } |
| |
| void Paeth16x64_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const ptrdiff_t stride16 = stride << 4; |
| const __m128i left_0 = LoadUnaligned16(left_column); |
| const __m128i top = LoadUnaligned16(top_row); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x16(dst, stride, top_left, top, left_0); |
| dst += stride16; |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i left_1 = LoadUnaligned16(left_ptr + 16); |
| WritePaeth16x16(dst, stride, top_left, top, left_1); |
| dst += stride16; |
| const __m128i left_2 = LoadUnaligned16(left_ptr + 32); |
| WritePaeth16x16(dst, stride, top_left, top, left_2); |
| dst += stride16; |
| const __m128i left_3 = LoadUnaligned16(left_ptr + 48); |
| WritePaeth16x16(dst, stride, top_left, top, left_3); |
| } |
| |
| void Paeth32x8_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i left = LoadLo8(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_0 = LoadUnaligned16(top_row); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* const dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x8(dst, stride, top_left, top_0, left); |
| const __m128i top_1 = LoadUnaligned16(top_ptr + 16); |
| WritePaeth16x8(dst + 16, stride, top_left, top_1, left); |
| } |
| |
| void Paeth32x16_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i left = LoadUnaligned16(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_0 = LoadUnaligned16(top_row); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* const dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x16(dst, stride, top_left, top_0, left); |
| const __m128i top_1 = LoadUnaligned16(top_ptr + 16); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left); |
| } |
| |
| void Paeth32x32_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i left_0 = LoadUnaligned16(left_ptr); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_0 = LoadUnaligned16(top_ptr); |
| const __m128i left_1 = LoadUnaligned16(left_ptr + 16); |
| const __m128i top_1 = LoadUnaligned16(top_ptr + 16); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_0); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_0); |
| dst += (stride << 4); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_1); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_1); |
| } |
| |
| void Paeth32x64_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i left_0 = LoadUnaligned16(left_ptr); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_0 = LoadUnaligned16(top_ptr); |
| const __m128i left_1 = LoadUnaligned16(left_ptr + 16); |
| const __m128i top_1 = LoadUnaligned16(top_ptr + 16); |
| const __m128i left_2 = LoadUnaligned16(left_ptr + 32); |
| const __m128i left_3 = LoadUnaligned16(left_ptr + 48); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_0); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_0); |
| dst += (stride << 4); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_1); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_1); |
| dst += (stride << 4); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_2); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_2); |
| dst += (stride << 4); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_3); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_3); |
| } |
| |
| void Paeth64x16_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const __m128i left = LoadUnaligned16(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_0 = LoadUnaligned16(top_ptr); |
| const __m128i top_1 = LoadUnaligned16(top_ptr + 16); |
| const __m128i top_2 = LoadUnaligned16(top_ptr + 32); |
| const __m128i top_3 = LoadUnaligned16(top_ptr + 48); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x16(dst, stride, top_left, top_0, left); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left); |
| WritePaeth16x16(dst + 32, stride, top_left, top_2, left); |
| WritePaeth16x16(dst + 48, stride, top_left, top_3, left); |
| } |
| |
| void Paeth64x32_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i left_0 = LoadUnaligned16(left_ptr); |
| const __m128i left_1 = LoadUnaligned16(left_ptr + 16); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_0 = LoadUnaligned16(top_ptr); |
| const __m128i top_1 = LoadUnaligned16(top_ptr + 16); |
| const __m128i top_2 = LoadUnaligned16(top_ptr + 32); |
| const __m128i top_3 = LoadUnaligned16(top_ptr + 48); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_0); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_0); |
| WritePaeth16x16(dst + 32, stride, top_left, top_2, left_0); |
| WritePaeth16x16(dst + 48, stride, top_left, top_3, left_0); |
| dst += (stride << 4); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_1); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_1); |
| WritePaeth16x16(dst + 32, stride, top_left, top_2, left_1); |
| WritePaeth16x16(dst + 48, stride, top_left, top_3, left_1); |
| } |
| |
| void Paeth64x64_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i left_0 = LoadUnaligned16(left_ptr); |
| const __m128i left_1 = LoadUnaligned16(left_ptr + 16); |
| const __m128i left_2 = LoadUnaligned16(left_ptr + 32); |
| const __m128i left_3 = LoadUnaligned16(left_ptr + 48); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_0 = LoadUnaligned16(top_ptr); |
| const __m128i top_1 = LoadUnaligned16(top_ptr + 16); |
| const __m128i top_2 = LoadUnaligned16(top_ptr + 32); |
| const __m128i top_3 = LoadUnaligned16(top_ptr + 48); |
| const uint8_t top_left = top_ptr[-1]; |
| auto* dst = static_cast<uint8_t*>(dest); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_0); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_0); |
| WritePaeth16x16(dst + 32, stride, top_left, top_2, left_0); |
| WritePaeth16x16(dst + 48, stride, top_left, top_3, left_0); |
| dst += (stride << 4); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_1); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_1); |
| WritePaeth16x16(dst + 32, stride, top_left, top_2, left_1); |
| WritePaeth16x16(dst + 48, stride, top_left, top_3, left_1); |
| dst += (stride << 4); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_2); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_2); |
| WritePaeth16x16(dst + 32, stride, top_left, top_2, left_2); |
| WritePaeth16x16(dst + 48, stride, top_left, top_3, left_2); |
| dst += (stride << 4); |
| WritePaeth16x16(dst, stride, top_left, top_0, left_3); |
| WritePaeth16x16(dst + 16, stride, top_left, top_1, left_3); |
| WritePaeth16x16(dst + 32, stride, top_left, top_2, left_3); |
| WritePaeth16x16(dst + 48, stride, top_left, top_3, left_3); |
| } |
| |
| //------------------------------------------------------------------------------ |
| // 7.11.2.4. Directional intra prediction process |
| |
| // Special case: An |xstep| of 64 corresponds to an angle delta of 45, meaning |
| // upsampling is ruled out. In addition, the bits masked by 0x3F for |
| // |shift_val| are 0 for all multiples of 64, so the formula |
| // val = top[top_base_x]*shift + top[top_base_x+1]*(32-shift), reduces to |
| // val = top[top_base_x+1] << 5, meaning only the second set of pixels is |
| // involved in the output. Hence |top| is offset by 1. |
| inline void DirectionalZone1_Step64(uint8_t* dst, ptrdiff_t stride, |
| const uint8_t* const top, const int width, |
| const int height) { |
| ptrdiff_t offset = 1; |
| if (height == 4) { |
| memcpy(dst, top + offset, width); |
| dst += stride; |
| memcpy(dst, top + offset + 1, width); |
| dst += stride; |
| memcpy(dst, top + offset + 2, width); |
| dst += stride; |
| memcpy(dst, top + offset + 3, width); |
| return; |
| } |
| int y = 0; |
| do { |
| memcpy(dst, top + offset, width); |
| dst += stride; |
| memcpy(dst, top + offset + 1, width); |
| dst += stride; |
| memcpy(dst, top + offset + 2, width); |
| dst += stride; |
| memcpy(dst, top + offset + 3, width); |
| dst += stride; |
| memcpy(dst, top + offset + 4, width); |
| dst += stride; |
| memcpy(dst, top + offset + 5, width); |
| dst += stride; |
| memcpy(dst, top + offset + 6, width); |
| dst += stride; |
| memcpy(dst, top + offset + 7, width); |
| dst += stride; |
| |
| offset += 8; |
| y += 8; |
| } while (y < height); |
| } |
| |
| inline void DirectionalZone1_4xH(uint8_t* dst, ptrdiff_t stride, |
| const uint8_t* const top, const int height, |
| const int xstep, const bool upsampled) { |
| const int upsample_shift = static_cast<int>(upsampled); |
| const int scale_bits = 6 - upsample_shift; |
| const int rounding_bits = 5; |
| const int max_base_x = (height + 3 /* width - 1 */) << upsample_shift; |
| const __m128i final_top_val = _mm_set1_epi16(top[max_base_x]); |
| const __m128i sampler = upsampled ? _mm_set_epi64x(0, 0x0706050403020100) |
| : _mm_set_epi64x(0, 0x0403030202010100); |
| // Each 16-bit value here corresponds to a position that may exceed |
| // |max_base_x|. When added to the top_base_x, it is used to mask values |
| // that pass the end of |top|. Starting from 1 to simulate "cmpge" which is |
| // not supported for packed integers. |
| const __m128i offsets = |
| _mm_set_epi32(0x00080007, 0x00060005, 0x00040003, 0x00020001); |
| |
| // All rows from |min_corner_only_y| down will simply use memcpy. |max_base_x| |
| // is always greater than |height|, so clipping to 1 is enough to make the |
| // logic work. |
| const int xstep_units = std::max(xstep >> scale_bits, 1); |
| const int min_corner_only_y = std::min(max_base_x / xstep_units, height); |
| |
| // Rows up to this y-value can be computed without checking for bounds. |
| int y = 0; |
| int top_x = xstep; |
| |
| for (; y < min_corner_only_y; ++y, dst += stride, top_x += xstep) { |
| const int top_base_x = top_x >> scale_bits; |
| |
| // Permit negative values of |top_x|. |
| const int shift_val = (LeftShift(top_x, upsample_shift) & 0x3F) >> 1; |
| const __m128i shift = _mm_set1_epi8(shift_val); |
| const __m128i max_shift = _mm_set1_epi8(32); |
| const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift); |
| const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift); |
| __m128i top_index_vect = _mm_set1_epi16(top_base_x); |
| top_index_vect = _mm_add_epi16(top_index_vect, offsets); |
| const __m128i max_base_x_vect = _mm_set1_epi16(max_base_x); |
| |
| // Load 8 values because we will select the sampled values based on |
| // |upsampled|. |
| const __m128i values = LoadLo8(top + top_base_x); |
| const __m128i sampled_values = _mm_shuffle_epi8(values, sampler); |
| const __m128i past_max = _mm_cmpgt_epi16(top_index_vect, max_base_x_vect); |
| __m128i prod = _mm_maddubs_epi16(sampled_values, shifts); |
| prod = RightShiftWithRounding_U16(prod, rounding_bits); |
| // Replace pixels from invalid range with top-right corner. |
| prod = _mm_blendv_epi8(prod, final_top_val, past_max); |
| Store4(dst, _mm_packus_epi16(prod, prod)); |
| } |
| |
| // Fill in corner-only rows. |
| for (; y < height; ++y) { |
| memset(dst, top[max_base_x], /* width */ 4); |
| dst += stride; |
| } |
| } |
| |
| // 7.11.2.4 (7) angle < 90 |
| inline void DirectionalZone1_Large(uint8_t* dest, ptrdiff_t stride, |
| const uint8_t* const top_row, |
| const int width, const int height, |
| const int xstep, const bool upsampled) { |
| const int upsample_shift = static_cast<int>(upsampled); |
| const __m128i sampler = |
| upsampled ? _mm_set_epi32(0x0F0E0D0C, 0x0B0A0908, 0x07060504, 0x03020100) |
| : _mm_set_epi32(0x08070706, 0x06050504, 0x04030302, 0x02010100); |
| const int scale_bits = 6 - upsample_shift; |
| const int max_base_x = ((width + height) - 1) << upsample_shift; |
| |
| const __m128i max_shift = _mm_set1_epi8(32); |
| const int rounding_bits = 5; |
| const int base_step = 1 << upsample_shift; |
| const int base_step8 = base_step << 3; |
| |
| // All rows from |min_corner_only_y| down will simply use memcpy. |max_base_x| |
| // is always greater than |height|, so clipping to 1 is enough to make the |
| // logic work. |
| const int xstep_units = std::max(xstep >> scale_bits, 1); |
| const int min_corner_only_y = std::min(max_base_x / xstep_units, height); |
| |
| // Rows up to this y-value can be computed without checking for bounds. |
| const int max_no_corner_y = std::min( |
| LeftShift((max_base_x - (base_step * width)), scale_bits) / xstep, |
| height); |
| // No need to check for exceeding |max_base_x| in the first loop. |
| int y = 0; |
| int top_x = xstep; |
| for (; y < max_no_corner_y; ++y, dest += stride, top_x += xstep) { |
| int top_base_x = top_x >> scale_bits; |
| // Permit negative values of |top_x|. |
| const int shift_val = (LeftShift(top_x, upsample_shift) & 0x3F) >> 1; |
| const __m128i shift = _mm_set1_epi8(shift_val); |
| const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift); |
| const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift); |
| int x = 0; |
| do { |
| const __m128i top_vals = LoadUnaligned16(top_row + top_base_x); |
| __m128i vals = _mm_shuffle_epi8(top_vals, sampler); |
| vals = _mm_maddubs_epi16(vals, shifts); |
| vals = RightShiftWithRounding_U16(vals, rounding_bits); |
| StoreLo8(dest + x, _mm_packus_epi16(vals, vals)); |
| top_base_x += base_step8; |
| x += 8; |
| } while (x < width); |
| } |
| |
| // Each 16-bit value here corresponds to a position that may exceed |
| // |max_base_x|. When added to the top_base_x, it is used to mask values |
| // that pass the end of |top|. Starting from 1 to simulate "cmpge" which is |
| // not supported for packed integers. |
| const __m128i offsets = |
| _mm_set_epi32(0x00080007, 0x00060005, 0x00040003, 0x00020001); |
| |
| const __m128i max_base_x_vect = _mm_set1_epi16(max_base_x); |
| const __m128i final_top_val = _mm_set1_epi16(top_row[max_base_x]); |
| const __m128i base_step8_vect = _mm_set1_epi16(base_step8); |
| for (; y < min_corner_only_y; ++y, dest += stride, top_x += xstep) { |
| int top_base_x = top_x >> scale_bits; |
| |
| const int shift_val = (LeftShift(top_x, upsample_shift) & 0x3F) >> 1; |
| const __m128i shift = _mm_set1_epi8(shift_val); |
| const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift); |
| const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift); |
| __m128i top_index_vect = _mm_set1_epi16(top_base_x); |
| top_index_vect = _mm_add_epi16(top_index_vect, offsets); |
| |
| int x = 0; |
| const int min_corner_only_x = |
| std::min(width, ((max_base_x - top_base_x) >> upsample_shift) + 7) & ~7; |
| for (; x < min_corner_only_x; |
| x += 8, top_base_x += base_step8, |
| top_index_vect = _mm_add_epi16(top_index_vect, base_step8_vect)) { |
| const __m128i past_max = _mm_cmpgt_epi16(top_index_vect, max_base_x_vect); |
| // Assuming a buffer zone of 8 bytes at the end of top_row, this prevents |
| // reading out of bounds. If all indices are past max and we don't need to |
| // use the loaded bytes at all, |top_base_x| becomes 0. |top_base_x| will |
| // reset for the next |y|. |
| top_base_x &= ~_mm_cvtsi128_si32(past_max); |
| const __m128i top_vals = LoadUnaligned16(top_row + top_base_x); |
| __m128i vals = _mm_shuffle_epi8(top_vals, sampler); |
| vals = _mm_maddubs_epi16(vals, shifts); |
| vals = RightShiftWithRounding_U16(vals, rounding_bits); |
| vals = _mm_blendv_epi8(vals, final_top_val, past_max); |
| StoreLo8(dest + x, _mm_packus_epi16(vals, vals)); |
| } |
| // Corner-only section of the row. |
| memset(dest + x, top_row[max_base_x], width - x); |
| } |
| // Fill in corner-only rows. |
| for (; y < height; ++y) { |
| memset(dest, top_row[max_base_x], width); |
| dest += stride; |
| } |
| } |
| |
| // 7.11.2.4 (7) angle < 90 |
| inline void DirectionalZone1_SSE4_1(uint8_t* dest, ptrdiff_t stride, |
| const uint8_t* const top_row, |
| const int width, const int height, |
| const int xstep, const bool upsampled) { |
| const int upsample_shift = static_cast<int>(upsampled); |
| if (xstep == 64) { |
| DirectionalZone1_Step64(dest, stride, top_row, width, height); |
| return; |
| } |
| if (width == 4) { |
| DirectionalZone1_4xH(dest, stride, top_row, height, xstep, upsampled); |
| return; |
| } |
| if (width >= 32) { |
| DirectionalZone1_Large(dest, stride, top_row, width, height, xstep, |
| upsampled); |
| return; |
| } |
| const __m128i sampler = |
| upsampled ? _mm_set_epi32(0x0F0E0D0C, 0x0B0A0908, 0x07060504, 0x03020100) |
| : _mm_set_epi32(0x08070706, 0x06050504, 0x04030302, 0x02010100); |
| const int scale_bits = 6 - upsample_shift; |
| const int max_base_x = ((width + height) - 1) << upsample_shift; |
| |
| const __m128i max_shift = _mm_set1_epi8(32); |
| const int rounding_bits = 5; |
| const int base_step = 1 << upsample_shift; |
| const int base_step8 = base_step << 3; |
| |
| // No need to check for exceeding |max_base_x| in the loops. |
| if (((xstep * height) >> scale_bits) + base_step * width < max_base_x) { |
| int top_x = xstep; |
| int y = 0; |
| do { |
| int top_base_x = top_x >> scale_bits; |
| // Permit negative values of |top_x|. |
| const int shift_val = (LeftShift(top_x, upsample_shift) & 0x3F) >> 1; |
| const __m128i shift = _mm_set1_epi8(shift_val); |
| const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift); |
| const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift); |
| int x = 0; |
| do { |
| const __m128i top_vals = LoadUnaligned16(top_row + top_base_x); |
| __m128i vals = _mm_shuffle_epi8(top_vals, sampler); |
| vals = _mm_maddubs_epi16(vals, shifts); |
| vals = RightShiftWithRounding_U16(vals, rounding_bits); |
| StoreLo8(dest + x, _mm_packus_epi16(vals, vals)); |
| top_base_x += base_step8; |
| x += 8; |
| } while (x < width); |
| dest += stride; |
| top_x += xstep; |
| } while (++y < height); |
| return; |
| } |
| |
| // Each 16-bit value here corresponds to a position that may exceed |
| // |max_base_x|. When added to the top_base_x, it is used to mask values |
| // that pass the end of |top|. Starting from 1 to simulate "cmpge" which is |
| // not supported for packed integers. |
| const __m128i offsets = |
| _mm_set_epi32(0x00080007, 0x00060005, 0x00040003, 0x00020001); |
| |
| const __m128i max_base_x_vect = _mm_set1_epi16(max_base_x); |
| const __m128i final_top_val = _mm_set1_epi16(top_row[max_base_x]); |
| const __m128i base_step8_vect = _mm_set1_epi16(base_step8); |
| int top_x = xstep; |
| int y = 0; |
| do { |
| int top_base_x = top_x >> scale_bits; |
| |
| if (top_base_x >= max_base_x) { |
| for (int i = y; i < height; ++i) { |
| memset(dest, top_row[max_base_x], width); |
| dest += stride; |
| } |
| return; |
| } |
| |
| const int shift_val = (LeftShift(top_x, upsample_shift) & 0x3F) >> 1; |
| const __m128i shift = _mm_set1_epi8(shift_val); |
| const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift); |
| const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift); |
| __m128i top_index_vect = _mm_set1_epi16(top_base_x); |
| top_index_vect = _mm_add_epi16(top_index_vect, offsets); |
| |
| int x = 0; |
| for (; x < width - 8; |
| x += 8, top_base_x += base_step8, |
| top_index_vect = _mm_add_epi16(top_index_vect, base_step8_vect)) { |
| const __m128i past_max = _mm_cmpgt_epi16(top_index_vect, max_base_x_vect); |
| // Assuming a buffer zone of 8 bytes at the end of top_row, this prevents |
| // reading out of bounds. If all indices are past max and we don't need to |
| // use the loaded bytes at all, |top_base_x| becomes 0. |top_base_x| will |
| // reset for the next |y|. |
| top_base_x &= ~_mm_cvtsi128_si32(past_max); |
| const __m128i top_vals = LoadUnaligned16(top_row + top_base_x); |
| __m128i vals = _mm_shuffle_epi8(top_vals, sampler); |
| vals = _mm_maddubs_epi16(vals, shifts); |
| vals = RightShiftWithRounding_U16(vals, rounding_bits); |
| vals = _mm_blendv_epi8(vals, final_top_val, past_max); |
| StoreLo8(dest + x, _mm_packus_epi16(vals, vals)); |
| } |
| const __m128i past_max = _mm_cmpgt_epi16(top_index_vect, max_base_x_vect); |
| __m128i vals; |
| if (upsampled) { |
| vals = LoadUnaligned16(top_row + top_base_x); |
| } else { |
| const __m128i top_vals = LoadLo8(top_row + top_base_x); |
| vals = _mm_shuffle_epi8(top_vals, sampler); |
| vals = _mm_insert_epi8(vals, top_row[top_base_x + 8], 15); |
| } |
| vals = _mm_maddubs_epi16(vals, shifts); |
| vals = RightShiftWithRounding_U16(vals, rounding_bits); |
| vals = _mm_blendv_epi8(vals, final_top_val, past_max); |
| StoreLo8(dest + x, _mm_packus_epi16(vals, vals)); |
| dest += stride; |
| top_x += xstep; |
| } while (++y < height); |
| } |
| |
| void DirectionalIntraPredictorZone1_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const int width, const int height, |
| const int xstep, |
| const bool upsampled_top) { |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| auto* dst = static_cast<uint8_t*>(dest); |
| DirectionalZone1_SSE4_1(dst, stride, top_ptr, width, height, xstep, |
| upsampled_top); |
| } |
| |
| template <bool upsampled> |
| inline void DirectionalZone3_4x4(uint8_t* dest, ptrdiff_t stride, |
| const uint8_t* const left_column, |
| const int base_left_y, const int ystep) { |
| // For use in the non-upsampled case. |
| const __m128i sampler = _mm_set_epi64x(0, 0x0403030202010100); |
| const int upsample_shift = static_cast<int>(upsampled); |
| const int scale_bits = 6 - upsample_shift; |
| const __m128i max_shift = _mm_set1_epi8(32); |
| const int rounding_bits = 5; |
| |
| __m128i result_block[4]; |
| for (int x = 0, left_y = base_left_y; x < 4; x++, left_y += ystep) { |
| const int left_base_y = left_y >> scale_bits; |
| const int shift_val = ((left_y << upsample_shift) & 0x3F) >> 1; |
| const __m128i shift = _mm_set1_epi8(shift_val); |
| const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift); |
| const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift); |
| __m128i vals; |
| if (upsampled) { |
| vals = LoadLo8(left_column + left_base_y); |
| } else { |
| const __m128i top_vals = LoadLo8(left_column + left_base_y); |
| vals = _mm_shuffle_epi8(top_vals, sampler); |
| } |
| vals = _mm_maddubs_epi16(vals, shifts); |
| vals = RightShiftWithRounding_U16(vals, rounding_bits); |
| result_block[x] = _mm_packus_epi16(vals, vals); |
| } |
| const __m128i result = Transpose4x4_U8(result_block); |
| // This is result_row0. |
| Store4(dest, result); |
| dest += stride; |
| const int result_row1 = _mm_extract_epi32(result, 1); |
| memcpy(dest, &result_row1, sizeof(result_row1)); |
| dest += stride; |
| const int result_row2 = _mm_extract_epi32(result, 2); |
| memcpy(dest, &result_row2, sizeof(result_row2)); |
| dest += stride; |
| const int result_row3 = _mm_extract_epi32(result, 3); |
| memcpy(dest, &result_row3, sizeof(result_row3)); |
| } |
| |
| template <bool upsampled, int height> |
| inline void DirectionalZone3_8xH(uint8_t* dest, ptrdiff_t stride, |
| const uint8_t* const left_column, |
| const int base_left_y, const int ystep) { |
| // For use in the non-upsampled case. |
| const __m128i sampler = |
| _mm_set_epi64x(0x0807070606050504, 0x0403030202010100); |
| const int upsample_shift = static_cast<int>(upsampled); |
| const int scale_bits = 6 - upsample_shift; |
| const __m128i max_shift = _mm_set1_epi8(32); |
| const int rounding_bits = 5; |
| |
| __m128i result_block[8]; |
| for (int x = 0, left_y = base_left_y; x < 8; x++, left_y += ystep) { |
| const int left_base_y = left_y >> scale_bits; |
| const int shift_val = (LeftShift(left_y, upsample_shift) & 0x3F) >> 1; |
| const __m128i shift = _mm_set1_epi8(shift_val); |
| const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift); |
| const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift); |
| __m128i vals; |
| if (upsampled) { |
| vals = LoadUnaligned16(left_column + left_base_y); |
| } else { |
| const __m128i top_vals = LoadUnaligned16(left_column + left_base_y); |
| vals = _mm_shuffle_epi8(top_vals, sampler); |
| } |
| vals = _mm_maddubs_epi16(vals, shifts); |
| result_block[x] = RightShiftWithRounding_U16(vals, rounding_bits); |
| } |
| Transpose8x8_U16(result_block, result_block); |
| for (int y = 0; y < height; ++y) { |
| StoreLo8(dest, _mm_packus_epi16(result_block[y], result_block[y])); |
| dest += stride; |
| } |
| } |
| |
| // 7.11.2.4 (9) angle > 180 |
| void DirectionalIntraPredictorZone3_SSE4_1(void* dest, ptrdiff_t stride, |
| const void* const left_column, |
| const int width, const int height, |
| const int ystep, |
| const bool upsampled) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const int upsample_shift = static_cast<int>(upsampled); |
| if (width == 4 || height == 4) { |
| const ptrdiff_t stride4 = stride << 2; |
| if (upsampled) { |
| int left_y = ystep; |
| int x = 0; |
| do { |
| uint8_t* dst_x = dst + x; |
| int y = 0; |
| do { |
| DirectionalZone3_4x4<true>( |
| dst_x, stride, left_ptr + (y << upsample_shift), left_y, ystep); |
| dst_x += stride4; |
| y += 4; |
| } while (y < height); |
| left_y += ystep << 2; |
| x += 4; |
| } while (x < width); |
| } else { |
| int left_y = ystep; |
| int x = 0; |
| do { |
| uint8_t* dst_x = dst + x; |
| int y = 0; |
| do { |
| DirectionalZone3_4x4<false>(dst_x, stride, left_ptr + y, left_y, |
| ystep); |
| dst_x += stride4; |
| y += 4; |
| } while (y < height); |
| left_y += ystep << 2; |
| x += 4; |
| } while (x < width); |
| } |
| return; |
| } |
| |
| const ptrdiff_t stride8 = stride << 3; |
| if (upsampled) { |
| int left_y = ystep; |
| int x = 0; |
| do { |
| uint8_t* dst_x = dst + x; |
| int y = 0; |
| do { |
| DirectionalZone3_8xH<true, 8>( |
| dst_x, stride, left_ptr + (y << upsample_shift), left_y, ystep); |
| dst_x += stride8; |
| y += 8; |
| } while (y < height); |
| left_y += ystep << 3; |
| x += 8; |
| } while (x < width); |
| } else { |
| int left_y = ystep; |
| int x = 0; |
| do { |
| uint8_t* dst_x = dst + x; |
| int y = 0; |
| do { |
| DirectionalZone3_8xH<false, 8>( |
| dst_x, stride, left_ptr + (y << upsample_shift), left_y, ystep); |
| dst_x += stride8; |
| y += 8; |
| } while (y < height); |
| left_y += ystep << 3; |
| x += 8; |
| } while (x < width); |
| } |
| } |
| |
| //------------------------------------------------------------------------------ |
| // Directional Zone 2 Functions |
| // 7.11.2.4 (8) |
| |
| // DirectionalBlend* selectively overwrites the values written by |
| // DirectionalZone2FromLeftCol*. |zone_bounds| has one 16-bit index for each |
| // row. |
| template <int y_selector> |
| inline void DirectionalBlend4_SSE4_1(uint8_t* dest, |
| const __m128i& dest_index_vect, |
| const __m128i& vals, |
| const __m128i& zone_bounds) { |
| const __m128i max_dest_x_vect = _mm_shufflelo_epi16(zone_bounds, y_selector); |
| const __m128i use_left = _mm_cmplt_epi16(dest_index_vect, max_dest_x_vect); |
| const __m128i original_vals = _mm_cvtepu8_epi16(Load4(dest)); |
| const __m128i blended_vals = _mm_blendv_epi8(vals, original_vals, use_left); |
| Store4(dest, _mm_packus_epi16(blended_vals, blended_vals)); |
| } |
| |
| inline void DirectionalBlend8_SSE4_1(uint8_t* dest, |
| const __m128i& dest_index_vect, |
| const __m128i& vals, |
| const __m128i& zone_bounds, |
| const __m128i& bounds_selector) { |
| const __m128i max_dest_x_vect = |
| _mm_shuffle_epi8(zone_bounds, bounds_selector); |
| const __m128i use_left = _mm_cmplt_epi16(dest_index_vect, max_dest_x_vect); |
| const __m128i original_vals = _mm_cvtepu8_epi16(LoadLo8(dest)); |
| const __m128i blended_vals = _mm_blendv_epi8(vals, original_vals, use_left); |
| StoreLo8(dest, _mm_packus_epi16(blended_vals, blended_vals)); |
| } |
| |
| constexpr int kDirectionalWeightBits = 5; |
| // |source| is packed with 4 or 8 pairs of 8-bit values from left or top. |
| // |shifts| is named to match the specification, with 4 or 8 pairs of (32 - |
| // shift) and shift. Shift is guaranteed to be between 0 and 32. |
| inline __m128i DirectionalZone2FromSource_SSE4_1(const uint8_t* const source, |
| const __m128i& shifts, |
| const __m128i& sampler) { |
| const __m128i src_vals = LoadUnaligned16(source); |
| __m128i vals = _mm_shuffle_epi8(src_vals, sampler); |
| vals = _mm_maddubs_epi16(vals, shifts); |
| return RightShiftWithRounding_U16(vals, kDirectionalWeightBits); |
| } |
| |
| // Because the source values "move backwards" as the row index increases, the |
| // indices derived from ystep are generally negative. This is accommodated by |
| // making sure the relative indices are within [-15, 0] when the function is |
| // called, and sliding them into the inclusive range [0, 15], relative to a |
| // lower base address. |
| constexpr int kPositiveIndexOffset = 15; |
| |
| template <bool upsampled> |
| inline void DirectionalZone2FromLeftCol_4x4_SSE4_1( |
| uint8_t* dst, ptrdiff_t stride, const uint8_t* const left_column_base, |
| __m128i left_y) { |
| const int upsample_shift = static_cast<int>(upsampled); |
| const int scale_bits = 6 - upsample_shift; |
| const __m128i max_shifts = _mm_set1_epi8(32); |
| const __m128i shift_mask = _mm_set1_epi32(0x003F003F); |
| const __m128i index_increment = _mm_cvtsi32_si128(0x01010101); |
| const __m128i positive_offset = _mm_set1_epi8(kPositiveIndexOffset); |
| // Left_column and sampler are both offset by 15 so the indices are always |
| // positive. |
| const uint8_t* left_column = left_column_base - kPositiveIndexOffset; |
| for (int y = 0; y < 4; dst += stride, ++y) { |
| __m128i offset_y = _mm_srai_epi16(left_y, scale_bits); |
| offset_y = _mm_packs_epi16(offset_y, offset_y); |
| |
| const __m128i adjacent = _mm_add_epi8(offset_y, index_increment); |
| __m128i sampler = _mm_unpacklo_epi8(offset_y, adjacent); |
| // Slide valid |offset_y| indices from range [-15, 0] to [0, 15] so they |
| // can work as shuffle indices. Some values may be out of bounds, but their |
| // pred results will be masked over by top prediction. |
| sampler = _mm_add_epi8(sampler, positive_offset); |
| |
| __m128i shifts = _mm_srli_epi16( |
| _mm_and_si128(_mm_slli_epi16(left_y, upsample_shift), shift_mask), 1); |
| shifts = _mm_packus_epi16(shifts, shifts); |
| const __m128i opposite_shifts = _mm_sub_epi8(max_shifts, shifts); |
| shifts = _mm_unpacklo_epi8(opposite_shifts, shifts); |
| const __m128i vals = DirectionalZone2FromSource_SSE4_1( |
| left_column + (y << upsample_shift), shifts, sampler); |
| Store4(dst, _mm_packus_epi16(vals, vals)); |
| } |
| } |
| |
| // The height at which a load of 16 bytes will not contain enough source pixels |
| // from |left_column| to supply an accurate row when computing 8 pixels at a |
| // time. The values are found by inspection. By coincidence, all angles that |
| // satisfy (ystep >> 6) == 2 map to the same value, so it is enough to look up |
| // by ystep >> 6. The largest index for this lookup is 1023 >> 6 == 15. |
| constexpr int kDirectionalZone2ShuffleInvalidHeight[16] = { |
| 1024, 1024, 16, 16, 16, 16, 0, 0, 18, 0, 0, 0, 0, 0, 0, 40}; |
| |
| template <bool upsampled> |
| inline void DirectionalZone2FromLeftCol_8x8_SSE4_1( |
| uint8_t* dst, ptrdiff_t stride, const uint8_t* const left_column, |
| __m128i left_y) { |
| const int upsample_shift = static_cast<int>(upsampled); |
| const int scale_bits = 6 - upsample_shift; |
| const __m128i max_shifts = _mm_set1_epi8(32); |
| const __m128i shift_mask = _mm_set1_epi32(0x003F003F); |
| const __m128i index_increment = _mm_set1_epi8(1); |
| const __m128i denegation = _mm_set1_epi8(kPositiveIndexOffset); |
| for (int y = 0; y < 8; dst += stride, ++y) { |
| __m128i offset_y = _mm_srai_epi16(left_y, scale_bits); |
| offset_y = _mm_packs_epi16(offset_y, offset_y); |
| const __m128i adjacent = _mm_add_epi8(offset_y, index_increment); |
| |
| // Offset the relative index because ystep is negative in Zone 2 and shuffle |
| // indices must be nonnegative. |
| __m128i sampler = _mm_unpacklo_epi8(offset_y, adjacent); |
| sampler = _mm_add_epi8(sampler, denegation); |
| |
| __m128i shifts = _mm_srli_epi16( |
| _mm_and_si128(_mm_slli_epi16(left_y, upsample_shift), shift_mask), 1); |
| shifts = _mm_packus_epi16(shifts, shifts); |
| const __m128i opposite_shifts = _mm_sub_epi8(max_shifts, shifts); |
| shifts = _mm_unpacklo_epi8(opposite_shifts, shifts); |
| |
| // The specification adds (y << 6) to left_y, which is subject to |
| // upsampling, but this puts sampler indices out of the 0-15 range. It is |
| // equivalent to offset the source address by (y << upsample_shift) instead. |
| const __m128i vals = DirectionalZone2FromSource_SSE4_1( |
| left_column - kPositiveIndexOffset + (y << upsample_shift), shifts, |
| sampler); |
| StoreLo8(dst, _mm_packus_epi16(vals, vals)); |
| } |
| } |
| |
| // |zone_bounds| is an epi16 of the relative x index at which base >= -(1 << |
| // upsampled_top), for each row. When there are 4 values, they can be duplicated |
| // with a non-register shuffle mask. |
| // |shifts| is one pair of weights that applies throughout a given row. |
| template <bool upsampled_top> |
| inline void DirectionalZone1Blend_4x4( |
| uint8_t* dest, const uint8_t* const top_row, ptrdiff_t stride, |
| __m128i sampler, const __m128i& zone_bounds, const __m128i& shifts, |
| const __m128i& dest_index_x, int top_x, const int xstep) { |
| const int upsample_shift = static_cast<int>(upsampled_top); |
| const int scale_bits_x = 6 - upsample_shift; |
| top_x -= xstep; |
| |
| int top_base_x = (top_x >> scale_bits_x); |
| const __m128i vals0 = DirectionalZone2FromSource_SSE4_1( |
| top_row + top_base_x, _mm_shufflelo_epi16(shifts, 0x00), sampler); |
| DirectionalBlend4_SSE4_1<0x00>(dest, dest_index_x, vals0, zone_bounds); |
| top_x -= xstep; |
| dest += stride; |
| |
| top_base_x = (top_x >> scale_bits_x); |
| const __m128i vals1 = DirectionalZone2FromSource_SSE4_1( |
| top_row + top_base_x, _mm_shufflelo_epi16(shifts, 0x55), sampler); |
| DirectionalBlend4_SSE4_1<0x55>(dest, dest_index_x, vals1, zone_bounds); |
| top_x -= xstep; |
| dest += stride; |
| |
| top_base_x = (top_x >> scale_bits_x); |
| const __m128i vals2 = DirectionalZone2FromSource_SSE4_1( |
| top_row + top_base_x, _mm_shufflelo_epi16(shifts, 0xAA), sampler); |
| DirectionalBlend4_SSE4_1<0xAA>(dest, dest_index_x, vals2, zone_bounds); |
| top_x -= xstep; |
| dest += stride; |
| |
| top_base_x = (top_x >> scale_bits_x); |
| const __m128i vals3 = DirectionalZone2FromSource_SSE4_1( |
| top_row + top_base_x, _mm_shufflelo_epi16(shifts, 0xFF), sampler); |
| DirectionalBlend4_SSE4_1<0xFF>(dest, dest_index_x, vals3, zone_bounds); |
| } |
| |
| template <bool upsampled_top, int height> |
| inline void DirectionalZone1Blend_8xH( |
| uint8_t* dest, const uint8_t* const top_row, ptrdiff_t stride, |
| __m128i sampler, const __m128i& zone_bounds, const __m128i& shifts, |
| const __m128i& dest_index_x, int top_x, const int xstep) { |
| const int upsample_shift = static_cast<int>(upsampled_top); |
| const int scale_bits_x = 6 - upsample_shift; |
| |
| __m128i y_selector = _mm_set1_epi32(0x01000100); |
| const __m128i index_increment = _mm_set1_epi32(0x02020202); |
| for (int y = 0; y < height; ++y, |
| y_selector = _mm_add_epi8(y_selector, index_increment), |
| dest += stride) { |
| top_x -= xstep; |
| const int top_base_x = top_x >> scale_bits_x; |
| const __m128i vals = DirectionalZone2FromSource_SSE4_1( |
| top_row + top_base_x, _mm_shuffle_epi8(shifts, y_selector), sampler); |
| DirectionalBlend8_SSE4_1(dest, dest_index_x, vals, zone_bounds, y_selector); |
| } |
| } |
| |
| // 7.11.2.4 (8) 90 < angle > 180 |
| // The strategy for this function is to know how many blocks can be processed |
| // with just pixels from |top_ptr|, then handle mixed blocks, then handle only |
| // blocks that take from |left_ptr|. Additionally, a fast index-shuffle |
| // approach is used for pred values from |left_column| in sections that permit |
| // it. |
| template <bool upsampled_left, bool upsampled_top> |
| inline void DirectionalZone2_SSE4_1(void* dest, ptrdiff_t stride, |
| const uint8_t* const top_row, |
| const uint8_t* const left_column, |
| const int width, const int height, |
| const int xstep, const int ystep) { |
| auto* dst = static_cast<uint8_t*>(dest); |
| const int upsample_left_shift = static_cast<int>(upsampled_left); |
| const int upsample_top_shift = static_cast<int>(upsampled_top); |
| const __m128i max_shift = _mm_set1_epi8(32); |
| const ptrdiff_t stride8 = stride << 3; |
| const __m128i dest_index_x = |
| _mm_set_epi32(0x00070006, 0x00050004, 0x00030002, 0x00010000); |
| const __m128i sampler_top = |
| upsampled_top |
| ? _mm_set_epi32(0x0F0E0D0C, 0x0B0A0908, 0x07060504, 0x03020100) |
| : _mm_set_epi32(0x08070706, 0x06050504, 0x04030302, 0x02010100); |
| const __m128i shift_mask = _mm_set1_epi32(0x003F003F); |
| // All columns from |min_top_only_x| to the right will only need |top_row| to |
| // compute. This assumes minimum |xstep| is 3. |
| const int min_top_only_x = std::min((height * xstep) >> 6, width); |
| |
| // For steep angles, the source pixels from left_column may not fit in a |
| // 16-byte load for shuffling. |
| // TODO(petersonab): Find a more precise formula for this subject to x. |
| const int max_shuffle_height = |
| std::min(height, kDirectionalZone2ShuffleInvalidHeight[ystep >> 6]); |
| |
| const int xstep8 = xstep << 3; |
| const __m128i xstep8_vect = _mm_set1_epi16(xstep8); |
| // Accumulate xstep across 8 rows. |
| const __m128i xstep_dup = _mm_set1_epi16(-xstep); |
| const __m128i increments = _mm_set_epi16(8, 7, 6, 5, 4, 3, 2, 1); |
| const __m128i xstep_for_shift = _mm_mullo_epi16(xstep_dup, increments); |
| // Offsets the original zone bound value to simplify x < (y+1)*xstep/64 -1 |
| const __m128i scaled_one = _mm_set1_epi16(-64); |
| __m128i xstep_bounds_base = |
| (xstep == 64) ? _mm_sub_epi16(scaled_one, xstep_for_shift) |
| : _mm_sub_epi16(_mm_set1_epi16(-1), xstep_for_shift); |
| |
| const int left_base_increment = ystep >> 6; |
| const int ystep_remainder = ystep & 0x3F; |
| const int ystep8 = ystep << 3; |
| const int left_base_increment8 = ystep8 >> 6; |
| const int ystep_remainder8 = ystep8 & 0x3F; |
| const __m128i increment_left8 = _mm_set1_epi16(-ystep_remainder8); |
| |
| // If the 64 scaling is regarded as a decimal point, the first value of the |
| // left_y vector omits the portion which is covered under the left_column |
| // offset. Following values need the full ystep as a relative offset. |
| const __m128i ystep_init = _mm_set1_epi16(-ystep_remainder); |
| const __m128i ystep_dup = _mm_set1_epi16(-ystep); |
| __m128i left_y = _mm_mullo_epi16(ystep_dup, dest_index_x); |
| left_y = _mm_add_epi16(ystep_init, left_y); |
| |
| const __m128i increment_top8 = _mm_set1_epi16(8 << 6); |
| int x = 0; |
| |
| // This loop treats each set of 4 columns in 3 stages with y-value boundaries. |
| // The first stage, before the first y-loop, covers blocks that are only |
| // computed from the top row. The second stage, comprising two y-loops, covers |
| // blocks that have a mixture of values computed from top or left. The final |
| // stage covers blocks that are only computed from the left. |
| for (int left_offset = -left_base_increment; x < min_top_only_x; |
| x += 8, |
| xstep_bounds_base = _mm_sub_epi16(xstep_bounds_base, increment_top8), |
| // Watch left_y because it can still get big. |
| left_y = _mm_add_epi16(left_y, increment_left8), |
| left_offset -= left_base_increment8) { |
| uint8_t* dst_x = dst + x; |
| |
| // Round down to the nearest multiple of 8. |
| const int max_top_only_y = std::min(((x + 1) << 6) / xstep, height) & ~7; |
| DirectionalZone1_4xH(dst_x, stride, top_row + (x << upsample_top_shift), |
| max_top_only_y, -xstep, upsampled_top); |
| DirectionalZone1_4xH(dst_x + 4, stride, |
| top_row + ((x + 4) << upsample_top_shift), |
| max_top_only_y, -xstep, upsampled_top); |
| |
| int y = max_top_only_y; |
| dst_x += stride * y; |
| const int xstep_y = xstep * y; |
| const __m128i xstep_y_vect = _mm_set1_epi16(xstep_y); |
| // All rows from |min_left_only_y| down for this set of columns, only need |
| // |left_column| to compute. |
| const int min_left_only_y = std::min(((x + 8) << 6) / xstep, height); |
| // At high angles such that min_left_only_y < 8, ystep is low and xstep is |
| // high. This means that max_shuffle_height is unbounded and xstep_bounds |
| // will overflow in 16 bits. This is prevented by stopping the first |
| // blending loop at min_left_only_y for such cases, which means we skip over |
| // the second blending loop as well. |
| const int left_shuffle_stop_y = |
| std::min(max_shuffle_height, min_left_only_y); |
| __m128i xstep_bounds = _mm_add_epi16(xstep_bounds_base, xstep_y_vect); |
| __m128i xstep_for_shift_y = _mm_sub_epi16(xstep_for_shift, xstep_y_vect); |
| int top_x = -xstep_y; |
| |
| for (; y < left_shuffle_stop_y; |
| y += 8, dst_x += stride8, |
| xstep_bounds = _mm_add_epi16(xstep_bounds, xstep8_vect), |
| xstep_for_shift_y = _mm_sub_epi16(xstep_for_shift_y, xstep8_vect), |
| top_x -= xstep8) { |
| DirectionalZone2FromLeftCol_8x8_SSE4_1<upsampled_left>( |
| dst_x, stride, |
| left_column + ((left_offset + y) << upsample_left_shift), left_y); |
| |
| __m128i shifts = _mm_srli_epi16( |
| _mm_and_si128(_mm_slli_epi16(xstep_for_shift_y, upsample_top_shift), |
| shift_mask), |
| 1); |
| shifts = _mm_packus_epi16(shifts, shifts); |
| __m128i opposite_shifts = _mm_sub_epi8(max_shift, shifts); |
| shifts = _mm_unpacklo_epi8(opposite_shifts, shifts); |
| __m128i xstep_bounds_off = _mm_srai_epi16(xstep_bounds, 6); |
| DirectionalZone1Blend_8xH<upsampled_top, 8>( |
| dst_x, top_row + (x << upsample_top_shift), stride, sampler_top, |
| xstep_bounds_off, shifts, dest_index_x, top_x, xstep); |
| } |
| // Pick up from the last y-value, using the 10% slower but secure method for |
| // left prediction. |
| const auto base_left_y = static_cast<int16_t>(_mm_extract_epi16(left_y, 0)); |
| for (; y < min_left_only_y; |
| y += 8, dst_x += stride8, |
| xstep_bounds = _mm_add_epi16(xstep_bounds, xstep8_vect), |
| xstep_for_shift_y = _mm_sub_epi16(xstep_for_shift_y, xstep8_vect), |
| top_x -= xstep8) { |
| const __m128i xstep_bounds_off = _mm_srai_epi16(xstep_bounds, 6); |
| |
| DirectionalZone3_8xH<upsampled_left, 8>( |
| dst_x, stride, |
| left_column + ((left_offset + y) << upsample_left_shift), base_left_y, |
| -ystep); |
| |
| __m128i shifts = _mm_srli_epi16( |
| _mm_and_si128(_mm_slli_epi16(xstep_for_shift_y, upsample_top_shift), |
| shift_mask), |
| 1); |
| shifts = _mm_packus_epi16(shifts, shifts); |
| __m128i opposite_shifts = _mm_sub_epi8(max_shift, shifts); |
| shifts = _mm_unpacklo_epi8(opposite_shifts, shifts); |
| DirectionalZone1Blend_8xH<upsampled_top, 8>( |
| dst_x, top_row + (x << upsample_top_shift), stride, sampler_top, |
| xstep_bounds_off, shifts, dest_index_x, top_x, xstep); |
| } |
| // Loop over y for left_only rows. |
| for (; y < height; y += 8, dst_x += stride8) { |
| DirectionalZone3_8xH<upsampled_left, 8>( |
| dst_x, stride, |
| left_column + ((left_offset + y) << upsample_left_shift), base_left_y, |
| -ystep); |
| } |
| } |
| for (; x < width; x += 4) { |
| DirectionalZone1_4xH(dst + x, stride, top_row + (x << upsample_top_shift), |
| height, -xstep, upsampled_top); |
| } |
| } |
| |
| template <bool upsampled_left, bool upsampled_top> |
| inline void DirectionalZone2_4_SSE4_1(void* dest, ptrdiff_t stride, |
| const uint8_t* const top_row, |
| const uint8_t* const left_column, |
| const int width, const int height, |
| const int xstep, const int ystep) { |
| auto* dst = static_cast<uint8_t*>(dest); |
| const int upsample_left_shift = static_cast<int>(upsampled_left); |
| const int upsample_top_shift = static_cast<int>(upsampled_top); |
| const __m128i max_shift = _mm_set1_epi8(32); |
| const ptrdiff_t stride4 = stride << 2; |
| const __m128i dest_index_x = _mm_set_epi32(0, 0, 0x00030002, 0x00010000); |
| const __m128i sampler_top = |
| upsampled_top |
| ? _mm_set_epi32(0x0F0E0D0C, 0x0B0A0908, 0x07060504, 0x03020100) |
| : _mm_set_epi32(0x08070706, 0x06050504, 0x04030302, 0x02010100); |
| // All columns from |min_top_only_x| to the right will only need |top_row| to |
| // compute. |
| assert(xstep >= 3); |
| const int min_top_only_x = std::min((height * xstep) >> 6, width); |
| |
| const int xstep4 = xstep << 2; |
| const __m128i xstep4_vect = _mm_set1_epi16(xstep4); |
| const __m128i xstep_dup = _mm_set1_epi16(-xstep); |
| const __m128i increments = _mm_set_epi32(0, 0, 0x00040003, 0x00020001); |
| __m128i xstep_for_shift = _mm_mullo_epi16(xstep_dup, increments); |
| const __m128i scaled_one = _mm_set1_epi16(-64); |
| // Offsets the original zone bound value to simplify x < (y+1)*xstep/64 -1 |
| __m128i xstep_bounds_base = |
| (xstep == 64) ? _mm_sub_epi16(scaled_one, xstep_for_shift) |
| : _mm_sub_epi16(_mm_set1_epi16(-1), xstep_for_shift); |
| |
| const int left_base_increment = ystep >> 6; |
| const int ystep_remainder = ystep & 0x3F; |
| const int ystep4 = ystep << 2; |
| const int left_base_increment4 = ystep4 >> 6; |
| // This is guaranteed to be less than 64, but accumulation may bring it past |
| // 64 for higher x values. |
| const int ystep_remainder4 = ystep4 & 0x3F; |
| const __m128i increment_left4 = _mm_set1_epi16(-ystep_remainder4); |
| const __m128i increment_top4 = _mm_set1_epi16(4 << 6); |
| |
| // If the 64 scaling is regarded as a decimal point, the first value of the |
| // left_y vector omits the portion which will go into the left_column offset. |
| // Following values need the full ystep as a relative offset. |
| const __m128i ystep_init = _mm_set1_epi16(-ystep_remainder); |
| const __m128i ystep_dup = _mm_set1_epi16(-ystep); |
| __m128i left_y = _mm_mullo_epi16(ystep_dup, dest_index_x); |
| left_y = _mm_add_epi16(ystep_init, left_y); |
| const __m128i shift_mask = _mm_set1_epi32(0x003F003F); |
| |
| int x = 0; |
| // Loop over x for columns with a mixture of sources. |
| for (int left_offset = -left_base_increment; x < min_top_only_x; x += 4, |
| xstep_bounds_base = _mm_sub_epi16(xstep_bounds_base, increment_top4), |
| left_y = _mm_add_epi16(left_y, increment_left4), |
| left_offset -= left_base_increment4) { |
| uint8_t* dst_x = dst + x; |
| |
| // Round down to the nearest multiple of 8. |
| const int max_top_only_y = std::min((x << 6) / xstep, height) & 0xFFFFFFF4; |
| DirectionalZone1_4xH(dst_x, stride, top_row + (x << upsample_top_shift), |
| max_top_only_y, -xstep, upsampled_top); |
| int y = max_top_only_y; |
| dst_x += stride * y; |
| const int xstep_y = xstep * y; |
| const __m128i xstep_y_vect = _mm_set1_epi16(xstep_y); |
| // All rows from |min_left_only_y| down for this set of columns, only need |
| // |left_column| to compute. Rounded up to the nearest multiple of 4. |
| const int min_left_only_y = std::min(((x + 4) << 6) / xstep, height); |
| |
| __m128i xstep_bounds = _mm_add_epi16(xstep_bounds_base, xstep_y_vect); |
| __m128i xstep_for_shift_y = _mm_sub_epi16(xstep_for_shift, xstep_y_vect); |
| int top_x = -xstep_y; |
| |
| // Loop over y for mixed rows. |
| for (; y < min_left_only_y; |
| y += 4, dst_x += stride4, |
| xstep_bounds = _mm_add_epi16(xstep_bounds, xstep4_vect), |
| xstep_for_shift_y = _mm_sub_epi16(xstep_for_shift_y, xstep4_vect), |
| top_x -= xstep4) { |
| DirectionalZone2FromLeftCol_4x4_SSE4_1<upsampled_left>( |
| dst_x, stride, |
| left_column + ((left_offset + y) * (1 << upsample_left_shift)), |
| left_y); |
| |
| __m128i shifts = _mm_srli_epi16( |
| _mm_and_si128(_mm_slli_epi16(xstep_for_shift_y, upsample_top_shift), |
| shift_mask), |
| 1); |
| shifts = _mm_packus_epi16(shifts, shifts); |
| const __m128i opposite_shifts = _mm_sub_epi8(max_shift, shifts); |
| shifts = _mm_unpacklo_epi8(opposite_shifts, shifts); |
| const __m128i xstep_bounds_off = _mm_srai_epi16(xstep_bounds, 6); |
| DirectionalZone1Blend_4x4<upsampled_top>( |
| dst_x, top_row + (x << upsample_top_shift), stride, sampler_top, |
| xstep_bounds_off, shifts, dest_index_x, top_x, xstep); |
| } |
| // Loop over y for left-only rows, if any. |
| for (; y < height; y += 4, dst_x += stride4) { |
| DirectionalZone2FromLeftCol_4x4_SSE4_1<upsampled_left>( |
| dst_x, stride, |
| left_column + ((left_offset + y) << upsample_left_shift), left_y); |
| } |
| } |
| // Loop over top-only columns, if any. |
| for (; x < width; x += 4) { |
| DirectionalZone1_4xH(dst + x, stride, top_row + (x << upsample_top_shift), |
| height, -xstep, upsampled_top); |
| } |
| } |
| |
| void DirectionalIntraPredictorZone2_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column, |
| const int width, const int height, |
| const int xstep, const int ystep, |
| const bool upsampled_top, |
| const bool upsampled_left) { |
| // Increasing the negative buffer for this function allows more rows to be |
| // processed at a time without branching in an inner loop to check the base. |
| uint8_t top_buffer[288]; |
| uint8_t left_buffer[288]; |
| memcpy(top_buffer + 128, static_cast<const uint8_t*>(top_row) - 16, 160); |
| memcpy(left_buffer + 128, static_cast<const uint8_t*>(left_column) - 16, 160); |
| const uint8_t* top_ptr = top_buffer + 144; |
| const uint8_t* left_ptr = left_buffer + 144; |
| if (width == 4 || height == 4) { |
| if (upsampled_left) { |
| if (upsampled_top) { |
| DirectionalZone2_4_SSE4_1<true, true>(dest, stride, top_ptr, left_ptr, |
| width, height, xstep, ystep); |
| } else { |
| DirectionalZone2_4_SSE4_1<true, false>(dest, stride, top_ptr, left_ptr, |
| width, height, xstep, ystep); |
| } |
| } else { |
| if (upsampled_top) { |
| DirectionalZone2_4_SSE4_1<false, true>(dest, stride, top_ptr, left_ptr, |
| width, height, xstep, ystep); |
| } else { |
| DirectionalZone2_4_SSE4_1<false, false>(dest, stride, top_ptr, left_ptr, |
| width, height, xstep, ystep); |
| } |
| } |
| return; |
| } |
| if (upsampled_left) { |
| if (upsampled_top) { |
| DirectionalZone2_SSE4_1<true, true>(dest, stride, top_ptr, left_ptr, |
| width, height, xstep, ystep); |
| } else { |
| DirectionalZone2_SSE4_1<true, false>(dest, stride, top_ptr, left_ptr, |
| width, height, xstep, ystep); |
| } |
| } else { |
| if (upsampled_top) { |
| DirectionalZone2_SSE4_1<false, true>(dest, stride, top_ptr, left_ptr, |
| width, height, xstep, ystep); |
| } else { |
| DirectionalZone2_SSE4_1<false, false>(dest, stride, top_ptr, left_ptr, |
| width, height, xstep, ystep); |
| } |
| } |
| } |
| |
| //------------------------------------------------------------------------------ |
| // FilterIntraPredictor_SSE4_1 |
| |
| // Apply all filter taps to the given 7 packed 16-bit values, keeping the 8th |
| // at zero to preserve the sum. |
| inline void Filter4x2_SSE4_1(uint8_t* dst, const ptrdiff_t stride, |
| const __m128i& pixels, const __m128i& taps_0_1, |
| const __m128i& taps_2_3, const __m128i& taps_4_5, |
| const __m128i& taps_6_7) { |
| const __m128i mul_0_01 = _mm_maddubs_epi16(pixels, taps_0_1); |
| const __m128i mul_0_23 = _mm_maddubs_epi16(pixels, taps_2_3); |
| // |output_half| contains 8 partial sums. |
| __m128i output_half = _mm_hadd_epi16(mul_0_01, mul_0_23); |
| __m128i output = _mm_hadd_epi16(output_half, output_half); |
| const __m128i output_row0 = |
| _mm_packus_epi16(RightShiftWithRounding_S16(output, 4), |
| /* arbitrary pack arg */ output); |
| Store4(dst, output_row0); |
| const __m128i mul_1_01 = _mm_maddubs_epi16(pixels, taps_4_5); |
| const __m128i mul_1_23 = _mm_maddubs_epi16(pixels, taps_6_7); |
| output_half = _mm_hadd_epi16(mul_1_01, mul_1_23); |
| output = _mm_hadd_epi16(output_half, output_half); |
| const __m128i output_row1 = |
| _mm_packus_epi16(RightShiftWithRounding_S16(output, 4), |
| /* arbitrary pack arg */ output); |
| Store4(dst + stride, output_row1); |
| } |
| |
| // 4xH transform sizes are given special treatment because LoadLo8 goes out |
| // of bounds and every block involves the left column. This implementation |
| // loads TL from the top row for the first block, so it is not |
| inline void Filter4xH(uint8_t* dest, ptrdiff_t stride, |
| const uint8_t* const top_ptr, |
| const uint8_t* const left_ptr, FilterIntraPredictor pred, |
| const int height) { |
| const __m128i taps_0_1 = LoadUnaligned16(kFilterIntraTaps[pred][0]); |
| const __m128i taps_2_3 = LoadUnaligned16(kFilterIntraTaps[pred][2]); |
| const __m128i taps_4_5 = LoadUnaligned16(kFilterIntraTaps[pred][4]); |
| const __m128i taps_6_7 = LoadUnaligned16(kFilterIntraTaps[pred][6]); |
| __m128i top = Load4(top_ptr - 1); |
| __m128i pixels = _mm_insert_epi8(top, top_ptr[3], 4); |
| __m128i left = (height == 4 ? Load4(left_ptr) : LoadLo8(left_ptr)); |
| left = _mm_slli_si128(left, 5); |
| |
| // Relative pixels: top[-1], top[0], top[1], top[2], top[3], left[0], left[1], |
| // left[2], left[3], left[4], left[5], left[6], left[7] |
| pixels = _mm_or_si128(left, pixels); |
| |
| // Duplicate first 8 bytes. |
| pixels = _mm_shuffle_epi32(pixels, kDuplicateFirstHalf); |
| Filter4x2_SSE4_1(dest, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| dest += stride; // Move to y = 1. |
| pixels = Load4(dest); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], empty, left[-2], left[-1], |
| // left[0], left[1], ... |
| pixels = _mm_or_si128(left, pixels); |
| |
| // This mask rearranges bytes in the order: 6, 0, 1, 2, 3, 7, 8, 15. The last |
| // byte is an unused value, which shall be multiplied by 0 when we apply the |
| // filter. |
| constexpr int64_t kInsertTopLeftFirstMask = 0x0F08070302010006; |
| |
| // Insert left[-1] in front as TL and put left[0] and left[1] at the end. |
| const __m128i pixel_order1 = _mm_set1_epi64x(kInsertTopLeftFirstMask); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 2. |
| Filter4x2_SSE4_1(dest, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| dest += stride; // Move to y = 3. |
| |
| // Compute the middle 8 rows before using common code for the final 4 rows. |
| // Because the common code below this block assumes that |
| if (height == 16) { |
| // This shift allows us to use pixel_order2 twice after shifting by 2 later. |
| left = _mm_slli_si128(left, 1); |
| pixels = Load4(dest); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], empty, empty, left[-4], |
| // left[-3], left[-2], left[-1], left[0], left[1], left[2], left[3] |
| pixels = _mm_or_si128(left, pixels); |
| |
| // This mask rearranges bytes in the order: 9, 0, 1, 2, 3, 7, 8, 15. The |
| // last byte is an unused value, as above. The top-left was shifted to |
| // position nine to keep two empty spaces after the top pixels. |
| constexpr int64_t kInsertTopLeftSecondMask = 0x0F0B0A0302010009; |
| |
| // Insert (relative) left[-1] in front as TL and put left[0] and left[1] at |
| // the end. |
| const __m128i pixel_order2 = _mm_set1_epi64x(kInsertTopLeftSecondMask); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| dest += stride; // Move to y = 4. |
| |
| // First 4x2 in the if body. |
| Filter4x2_SSE4_1(dest, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| |
| // Clear all but final pixel in the first 8 of left column. |
| __m128i keep_top_left = _mm_srli_si128(left, 13); |
| dest += stride; // Move to y = 5. |
| pixels = Load4(dest); |
| left = _mm_srli_si128(left, 2); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], left[-6], |
| // left[-5], left[-4], left[-3], left[-2], left[-1], left[0], left[1] |
| pixels = _mm_or_si128(left, pixels); |
| left = LoadLo8(left_ptr + 8); |
| |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| dest += stride; // Move to y = 6. |
| |
| // Second 4x2 in the if body. |
| Filter4x2_SSE4_1(dest, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| |
| // Position TL value so we can use pixel_order1. |
| keep_top_left = _mm_slli_si128(keep_top_left, 6); |
| dest += stride; // Move to y = 7. |
| pixels = Load4(dest); |
| left = _mm_slli_si128(left, 7); |
| left = _mm_or_si128(left, keep_top_left); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], empty, empty, |
| // left[-1], left[0], left[1], left[2], left[3], ... |
| pixels = _mm_or_si128(left, pixels); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 8. |
| |
| // Third 4x2 in the if body. |
| Filter4x2_SSE4_1(dest, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| dest += stride; // Move to y = 9. |
| |
| // Prepare final inputs. |
| pixels = Load4(dest); |
| left = _mm_srli_si128(left, 2); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], left[-3], left[-2] |
| // left[-1], left[0], left[1], left[2], left[3], ... |
| pixels = _mm_or_si128(left, pixels); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 10. |
| |
| // Fourth 4x2 in the if body. |
| Filter4x2_SSE4_1(dest, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| dest += stride; // Move to y = 11. |
| } |
| |
| // In both the 8 and 16 case, we assume that the left vector has the next TL |
| // at position 8. |
| if (height > 4) { |
| // Erase prior left pixels by shifting TL to position 0. |
| left = _mm_srli_si128(left, 8); |
| left = _mm_slli_si128(left, 6); |
| pixels = Load4(dest); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], empty, empty, |
| // left[-1], left[0], left[1], left[2], left[3], ... |
| pixels = _mm_or_si128(left, pixels); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 12 or 4. |
| |
| // First of final two 4x2 blocks. |
| Filter4x2_SSE4_1(dest, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| dest += stride; // Move to y = 13 or 5. |
| pixels = Load4(dest); |
| left = _mm_srli_si128(left, 2); |
| |
| // Relative pixels: top[0], top[1], top[2], top[3], left[-3], left[-2] |
| // left[-1], left[0], left[1], left[2], left[3], ... |
| pixels = _mm_or_si128(left, pixels); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| dest += stride; // Move to y = 14 or 6. |
| |
| // Last of final two 4x2 blocks. |
| Filter4x2_SSE4_1(dest, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| } |
| } |
| |
| void FilterIntraPredictor_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column, |
| FilterIntraPredictor pred, const int width, |
| const int height) { |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| if (width == 4) { |
| Filter4xH(dst, stride, top_ptr, left_ptr, pred, height); |
| return; |
| } |
| |
| // There is one set of 7 taps for each of the 4x2 output pixels. |
| const __m128i taps_0_1 = LoadUnaligned16(kFilterIntraTaps[pred][0]); |
| const __m128i taps_2_3 = LoadUnaligned16(kFilterIntraTaps[pred][2]); |
| const __m128i taps_4_5 = LoadUnaligned16(kFilterIntraTaps[pred][4]); |
| const __m128i taps_6_7 = LoadUnaligned16(kFilterIntraTaps[pred][6]); |
| |
| // This mask rearranges bytes in the order: 0, 1, 2, 3, 4, 8, 9, 15. The 15 at |
| // the end is an unused value, which shall be multiplied by 0 when we apply |
| // the filter. |
| constexpr int64_t kCondenseLeftMask = 0x0F09080403020100; |
| |
| // Takes the "left section" and puts it right after p0-p4. |
| const __m128i pixel_order1 = _mm_set1_epi64x(kCondenseLeftMask); |
| |
| // This mask rearranges bytes in the order: 8, 0, 1, 2, 3, 9, 10, 15. The last |
| // byte is unused as above. |
| constexpr int64_t kInsertTopLeftMask = 0x0F0A090302010008; |
| |
| // Shuffles the "top left" from the left section, to the front. Used when |
| // grabbing data from left_column and not top_row. |
| const __m128i pixel_order2 = _mm_set1_epi64x(kInsertTopLeftMask); |
| |
| // This first pass takes care of the cases where the top left pixel comes from |
| // top_row. |
| __m128i pixels = LoadLo8(top_ptr - 1); |
| __m128i left = _mm_slli_si128(Load4(left_column), 8); |
| pixels = _mm_or_si128(pixels, left); |
| |
| // Two sets of the same pixels to multiply with two sets of taps. |
| pixels = _mm_shuffle_epi8(pixels, pixel_order1); |
| Filter4x2_SSE4_1(dst, stride, pixels, taps_0_1, taps_2_3, taps_4_5, taps_6_7); |
| left = _mm_srli_si128(left, 1); |
| |
| // Load |
| pixels = Load4(dst + stride); |
| |
| // Because of the above shift, this OR 'invades' the final of the first 8 |
| // bytes of |pixels|. This is acceptable because the 8th filter tap is always |
| // a padded 0. |
| pixels = _mm_or_si128(pixels, left); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| const ptrdiff_t stride2 = stride << 1; |
| const ptrdiff_t stride4 = stride << 2; |
| Filter4x2_SSE4_1(dst + stride2, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| dst += 4; |
| for (int x = 3; x < width - 4; x += 4) { |
| pixels = Load4(top_ptr + x); |
| pixels = _mm_insert_epi8(pixels, top_ptr[x + 4], 4); |
| pixels = _mm_insert_epi8(pixels, dst[-1], 5); |
| pixels = _mm_insert_epi8(pixels, dst[stride - 1], 6); |
| |
| // Duplicate bottom half into upper half. |
| pixels = _mm_shuffle_epi32(pixels, kDuplicateFirstHalf); |
| Filter4x2_SSE4_1(dst, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| pixels = Load4(dst + stride - 1); |
| pixels = _mm_insert_epi8(pixels, dst[stride + 3], 4); |
| pixels = _mm_insert_epi8(pixels, dst[stride2 - 1], 5); |
| pixels = _mm_insert_epi8(pixels, dst[stride + stride2 - 1], 6); |
| |
| // Duplicate bottom half into upper half. |
| pixels = _mm_shuffle_epi32(pixels, kDuplicateFirstHalf); |
| Filter4x2_SSE4_1(dst + stride2, stride, pixels, taps_0_1, taps_2_3, |
| taps_4_5, taps_6_7); |
| dst += 4; |
| } |
| |
| // Now we handle heights that reference previous blocks rather than top_row. |
| for (int y = 4; y < height; y += 4) { |
| // Leftmost 4x4 block for this height. |
| dst -= width; |
| dst += stride4; |
| |
| // Top Left is not available by offset in these leftmost blocks. |
| pixels = Load4(dst - stride); |
| left = _mm_slli_si128(Load4(left_ptr + y - 1), 8); |
| left = _mm_insert_epi8(left, left_ptr[y + 3], 12); |
| pixels = _mm_or_si128(pixels, left); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| Filter4x2_SSE4_1(dst, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| |
| // The bytes shifted into positions 6 and 7 will be ignored by the shuffle. |
| left = _mm_srli_si128(left, 2); |
| pixels = Load4(dst + stride); |
| pixels = _mm_or_si128(pixels, left); |
| pixels = _mm_shuffle_epi8(pixels, pixel_order2); |
| Filter4x2_SSE4_1(dst + stride2, stride, pixels, taps_0_1, taps_2_3, |
| taps_4_5, taps_6_7); |
| |
| dst += 4; |
| |
| // Remaining 4x4 blocks for this height. |
| for (int x = 4; x < width; x += 4) { |
| pixels = Load4(dst - stride - 1); |
| pixels = _mm_insert_epi8(pixels, dst[-stride + 3], 4); |
| pixels = _mm_insert_epi8(pixels, dst[-1], 5); |
| pixels = _mm_insert_epi8(pixels, dst[stride - 1], 6); |
| |
| // Duplicate bottom half into upper half. |
| pixels = _mm_shuffle_epi32(pixels, kDuplicateFirstHalf); |
| Filter4x2_SSE4_1(dst, stride, pixels, taps_0_1, taps_2_3, taps_4_5, |
| taps_6_7); |
| pixels = Load4(dst + stride - 1); |
| pixels = _mm_insert_epi8(pixels, dst[stride + 3], 4); |
| pixels = _mm_insert_epi8(pixels, dst[stride2 - 1], 5); |
| pixels = _mm_insert_epi8(pixels, dst[stride2 + stride - 1], 6); |
| |
| // Duplicate bottom half into upper half. |
| pixels = _mm_shuffle_epi32(pixels, kDuplicateFirstHalf); |
| Filter4x2_SSE4_1(dst + stride2, stride, pixels, taps_0_1, taps_2_3, |
| taps_4_5, taps_6_7); |
| dst += 4; |
| } |
| } |
| } |
| |
| void Init8bpp() { |
| Dsp* const dsp = dsp_internal::GetWritableDspTable(kBitdepth8); |
| assert(dsp != nullptr); |
| static_cast<void>(dsp); |
| // These guards check if this version of the function was not superseded by |
| // a higher optimization level, such as AVX. The corresponding #define also |
| // prevents the C version from being added to the table. |
| #if DSP_ENABLED_8BPP_SSE4_1(FilterIntraPredictor) |
| dsp->filter_intra_predictor = FilterIntraPredictor_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(DirectionalIntraPredictorZone1) |
| dsp->directional_intra_predictor_zone1 = |
| DirectionalIntraPredictorZone1_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(DirectionalIntraPredictorZone2) |
| dsp->directional_intra_predictor_zone2 = |
| DirectionalIntraPredictorZone2_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(DirectionalIntraPredictorZone3) |
| dsp->directional_intra_predictor_zone3 = |
| DirectionalIntraPredictorZone3_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorDcTop] = |
| DcDefs::_4x4::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorDcTop] = |
| DcDefs::_4x8::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorDcTop] = |
| DcDefs::_4x16::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorDcTop] = |
| DcDefs::_8x4::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorDcTop] = |
| DcDefs::_8x8::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorDcTop] = |
| DcDefs::_8x16::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorDcTop] = |
| DcDefs::_8x32::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorDcTop] = |
| DcDefs::_16x4::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorDcTop] = |
| DcDefs::_16x8::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorDcTop] = |
| DcDefs::_16x16::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorDcTop] = |
| DcDefs::_16x32::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x64_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorDcTop] = |
| DcDefs::_16x64::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorDcTop] = |
| DcDefs::_32x8::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorDcTop] = |
| DcDefs::_32x16::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorDcTop] = |
| DcDefs::_32x32::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x64_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorDcTop] = |
| DcDefs::_32x64::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x16_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorDcTop] = |
| DcDefs::_64x16::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x32_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorDcTop] = |
| DcDefs::_64x32::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x64_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorDcTop] = |
| DcDefs::_64x64::DcTop; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorDcLeft] = |
| DcDefs::_4x4::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorDcLeft] = |
| DcDefs::_4x8::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorDcLeft] = |
| DcDefs::_4x16::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorDcLeft] = |
| DcDefs::_8x4::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorDcLeft] = |
| DcDefs::_8x8::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorDcLeft] = |
| DcDefs::_8x16::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorDcLeft] = |
| DcDefs::_8x32::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorDcLeft] = |
| DcDefs::_16x4::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorDcLeft] = |
| DcDefs::_16x8::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorDcLeft] = |
| DcDefs::_16x16::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorDcLeft] = |
| DcDefs::_16x32::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x64_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorDcLeft] = |
| DcDefs::_16x64::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorDcLeft] = |
| DcDefs::_32x8::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorDcLeft] = |
| DcDefs::_32x16::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorDcLeft] = |
| DcDefs::_32x32::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x64_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorDcLeft] = |
| DcDefs::_32x64::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x16_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorDcLeft] = |
| DcDefs::_64x16::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x32_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorDcLeft] = |
| DcDefs::_64x32::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x64_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorDcLeft] = |
| DcDefs::_64x64::DcLeft; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorDc] = |
| DcDefs::_4x4::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorDc] = |
| DcDefs::_4x8::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorDc] = |
| DcDefs::_4x16::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorDc] = |
| DcDefs::_8x4::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorDc] = |
| DcDefs::_8x8::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorDc] = |
| DcDefs::_8x16::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorDc] = |
| DcDefs::_8x32::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorDc] = |
| DcDefs::_16x4::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorDc] = |
| DcDefs::_16x8::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorDc] = |
| DcDefs::_16x16::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorDc] = |
| DcDefs::_16x32::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x64_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorDc] = |
| DcDefs::_16x64::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorDc] = |
| DcDefs::_32x8::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorDc] = |
| DcDefs::_32x16::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorDc] = |
| DcDefs::_32x32::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x64_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorDc] = |
| DcDefs::_32x64::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x16_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorDc] = |
| DcDefs::_64x16::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x32_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorDc] = |
| DcDefs::_64x32::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x64_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorDc] = |
| DcDefs::_64x64::Dc; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorPaeth] = |
| Paeth4x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorPaeth] = |
| Paeth4x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorPaeth] = |
| Paeth4x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorPaeth] = |
| Paeth8x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorPaeth] = |
| Paeth8x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorPaeth] = |
| Paeth8x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorPaeth] = |
| Paeth8x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorPaeth] = |
| Paeth16x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorPaeth] = |
| Paeth16x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorPaeth] = |
| Paeth16x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorPaeth] = |
| Paeth16x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x64_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorPaeth] = |
| Paeth16x64_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorPaeth] = |
| Paeth32x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorPaeth] = |
| Paeth32x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorPaeth] = |
| Paeth32x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x64_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorPaeth] = |
| Paeth32x64_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x16_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorPaeth] = |
| Paeth64x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x32_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorPaeth] = |
| Paeth64x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x64_IntraPredictorPaeth) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorPaeth] = |
| Paeth64x64_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorHorizontal] = |
| DirDefs::_4x4::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorHorizontal] = |
| DirDefs::_4x8::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorHorizontal] = |
| DirDefs::_4x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorHorizontal] = |
| DirDefs::_8x4::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorHorizontal] = |
| DirDefs::_8x8::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorHorizontal] = |
| DirDefs::_8x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorHorizontal] = |
| DirDefs::_8x32::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorHorizontal] = |
| DirDefs::_16x4::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorHorizontal] = |
| DirDefs::_16x8::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorHorizontal] = |
| DirDefs::_16x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorHorizontal] = |
| DirDefs::_16x32::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x64_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorHorizontal] = |
| DirDefs::_16x64::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorHorizontal] = |
| DirDefs::_32x8::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorHorizontal] = |
| DirDefs::_32x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorHorizontal] = |
| DirDefs::_32x32::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x64_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorHorizontal] = |
| DirDefs::_32x64::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorHorizontal] = |
| DirDefs::_64x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x32_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorHorizontal] = |
| DirDefs::_64x32::Horizontal; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x64_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorHorizontal] = |
| DirDefs::_64x64::Horizontal; |
| #endif |
| } // NOLINT(readability/fn_size) |
| // TODO(petersonab): Split Init8bpp function into family-specific files. |
| |
| } // namespace |
| } // namespace low_bitdepth |
| |
| //------------------------------------------------------------------------------ |
| #if LIBGAV1_MAX_BITDEPTH >= 10 |
| namespace high_bitdepth { |
| namespace { |
| |
| template <int height> |
| inline void DcStore4xH_SSE4_1(void* const dest, ptrdiff_t stride, |
| const __m128i dc) { |
| const __m128i dc_dup = _mm_shufflelo_epi16(dc, 0); |
| int y = height - 1; |
| auto* dst = static_cast<uint8_t*>(dest); |
| do { |
| StoreLo8(dst, dc_dup); |
| dst += stride; |
| } while (--y != 0); |
| StoreLo8(dst, dc_dup); |
| } |
| |
| // WriteDuplicateN assumes dup has 4 32-bit "units," each of which comprises 2 |
| // identical shorts that need N total copies written into dest. The unpacking |
| // works the same as in the 8bpp case, except that each 32-bit unit needs twice |
| // as many copies. |
| inline void WriteDuplicate4x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| auto* dst = static_cast<uint8_t*>(dest); |
| _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), dup64_lo); |
| dst += stride; |
| _mm_storeh_pi(reinterpret_cast<__m64*>(dst), _mm_castsi128_ps(dup64_lo)); |
| dst += stride; |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), dup64_hi); |
| dst += stride; |
| _mm_storeh_pi(reinterpret_cast<__m64*>(dst), _mm_castsi128_ps(dup64_hi)); |
| } |
| |
| inline void WriteDuplicate8x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i dup128_0 = _mm_unpacklo_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_0); |
| dst += stride; |
| const __m128i dup128_1 = _mm_unpackhi_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_1); |
| dst += stride; |
| const __m128i dup128_2 = _mm_unpacklo_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_2); |
| dst += stride; |
| const __m128i dup128_3 = _mm_unpackhi_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_3); |
| } |
| |
| inline void WriteDuplicate16x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i dup128_0 = _mm_unpacklo_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_0); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_0); |
| dst += stride; |
| const __m128i dup128_1 = _mm_unpackhi_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_1); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_1); |
| dst += stride; |
| const __m128i dup128_2 = _mm_unpacklo_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_2); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_2); |
| dst += stride; |
| const __m128i dup128_3 = _mm_unpackhi_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_3); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_3); |
| } |
| |
| inline void WriteDuplicate32x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i dup128_0 = _mm_unpacklo_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_0); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_0); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), dup128_0); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), dup128_0); |
| dst += stride; |
| const __m128i dup128_1 = _mm_unpackhi_epi64(dup64_lo, dup64_lo); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_1); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_1); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), dup128_1); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), dup128_1); |
| dst += stride; |
| const __m128i dup128_2 = _mm_unpacklo_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_2); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_2); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), dup128_2); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), dup128_2); |
| dst += stride; |
| const __m128i dup128_3 = _mm_unpackhi_epi64(dup64_hi, dup64_hi); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), dup128_3); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 16), dup128_3); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 32), dup128_3); |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + 48), dup128_3); |
| } |
| |
| inline void WriteDuplicate64x4(void* const dest, ptrdiff_t stride, |
| const __m128i dup32) { |
| const __m128i dup64_lo = _mm_unpacklo_epi32(dup32, dup32); |
| const __m128i dup64_hi = _mm_unpackhi_epi32(dup32, dup32); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i dup128_0 = _mm_unpacklo_epi64(dup64_lo, dup64_lo); |
| for (int x = 0; x < 128; x += 16) { |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + x), dup128_0); |
| } |
| dst += stride; |
| const __m128i dup128_1 = _mm_unpackhi_epi64(dup64_lo, dup64_lo); |
| for (int x = 0; x < 128; x += 16) { |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + x), dup128_1); |
| } |
| dst += stride; |
| const __m128i dup128_2 = _mm_unpacklo_epi64(dup64_hi, dup64_hi); |
| for (int x = 0; x < 128; x += 16) { |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + x), dup128_2); |
| } |
| dst += stride; |
| const __m128i dup128_3 = _mm_unpackhi_epi64(dup64_hi, dup64_hi); |
| for (int x = 0; x < 128; x += 16) { |
| _mm_storeu_si128(reinterpret_cast<__m128i*>(dst + x), dup128_3); |
| } |
| } |
| |
| // ColStoreN<height> copies each of the |height| values in |column| across its |
| // corresponding row in dest. |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore4_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const __m128i col_data = LoadLo8(column); |
| const __m128i col_dup32 = _mm_unpacklo_epi16(col_data, col_data); |
| writefn(dest, stride, col_dup32); |
| } |
| |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore8_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const __m128i col_data = LoadUnaligned16(column); |
| const __m128i col_dup32_lo = _mm_unpacklo_epi16(col_data, col_data); |
| const __m128i col_dup32_hi = _mm_unpackhi_epi16(col_data, col_data); |
| auto* dst = static_cast<uint8_t*>(dest); |
| writefn(dst, stride, col_dup32_lo); |
| const ptrdiff_t stride4 = stride << 2; |
| dst += stride4; |
| writefn(dst, stride, col_dup32_hi); |
| } |
| |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore16_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const ptrdiff_t stride4 = stride << 2; |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y = 0; y < 32; y += 16) { |
| const __m128i col_data = |
| LoadUnaligned16(static_cast<const uint8_t*>(column) + y); |
| const __m128i col_dup32_lo = _mm_unpacklo_epi16(col_data, col_data); |
| const __m128i col_dup32_hi = _mm_unpackhi_epi16(col_data, col_data); |
| writefn(dst, stride, col_dup32_lo); |
| dst += stride4; |
| writefn(dst, stride, col_dup32_hi); |
| dst += stride4; |
| } |
| } |
| |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore32_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const ptrdiff_t stride4 = stride << 2; |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y = 0; y < 64; y += 16) { |
| const __m128i col_data = |
| LoadUnaligned16(static_cast<const uint8_t*>(column) + y); |
| const __m128i col_dup32_lo = _mm_unpacklo_epi16(col_data, col_data); |
| const __m128i col_dup32_hi = _mm_unpackhi_epi16(col_data, col_data); |
| writefn(dst, stride, col_dup32_lo); |
| dst += stride4; |
| writefn(dst, stride, col_dup32_hi); |
| dst += stride4; |
| } |
| } |
| |
| template <WriteDuplicateFunc writefn> |
| inline void ColStore64_SSE4_1(void* const dest, ptrdiff_t stride, |
| const void* const column) { |
| const ptrdiff_t stride4 = stride << 2; |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y = 0; y < 128; y += 16) { |
| const __m128i col_data = |
| LoadUnaligned16(static_cast<const uint8_t*>(column) + y); |
| const __m128i col_dup32_lo = _mm_unpacklo_epi16(col_data, col_data); |
| const __m128i col_dup32_hi = _mm_unpackhi_epi16(col_data, col_data); |
| writefn(dst, stride, col_dup32_lo); |
| dst += stride4; |
| writefn(dst, stride, col_dup32_hi); |
| dst += stride4; |
| } |
| } |
| |
| // |ref| points to 8 bytes containing 4 packed int16 values. |
| inline __m128i DcSum4_SSE4_1(const void* ref) { |
| const __m128i vals = _mm_loadl_epi64(static_cast<const __m128i*>(ref)); |
| const __m128i ones = _mm_set1_epi16(1); |
| |
| // half_sum[31:0] = a1+a2 |
| // half_sum[63:32] = a3+a4 |
| const __m128i half_sum = _mm_madd_epi16(vals, ones); |
| // Place half_sum[63:32] in shift_sum[31:0]. |
| const __m128i shift_sum = _mm_srli_si128(half_sum, 4); |
| return _mm_add_epi32(half_sum, shift_sum); |
| } |
| |
| struct DcDefs { |
| DcDefs() = delete; |
| |
| using _4x4 = DcPredFuncs_SSE4_1<2, 2, DcSum4_SSE4_1, DcSum4_SSE4_1, |
| DcStore4xH_SSE4_1<4>, 0, 0>; |
| }; |
| |
| struct DirDefs { |
| DirDefs() = delete; |
| |
| using _4x4 = DirectionalPredFuncs_SSE4_1<ColStore4_SSE4_1<WriteDuplicate4x4>>; |
| using _4x8 = DirectionalPredFuncs_SSE4_1<ColStore8_SSE4_1<WriteDuplicate4x4>>; |
| using _4x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate4x4>>; |
| using _8x4 = DirectionalPredFuncs_SSE4_1<ColStore4_SSE4_1<WriteDuplicate8x4>>; |
| using _8x8 = DirectionalPredFuncs_SSE4_1<ColStore8_SSE4_1<WriteDuplicate8x4>>; |
| using _8x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate8x4>>; |
| using _8x32 = |
| DirectionalPredFuncs_SSE4_1<ColStore32_SSE4_1<WriteDuplicate8x4>>; |
| using _16x4 = |
| DirectionalPredFuncs_SSE4_1<ColStore4_SSE4_1<WriteDuplicate16x4>>; |
| using _16x8 = |
| DirectionalPredFuncs_SSE4_1<ColStore8_SSE4_1<WriteDuplicate16x4>>; |
| using _16x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate16x4>>; |
| using _16x32 = |
| DirectionalPredFuncs_SSE4_1<ColStore32_SSE4_1<WriteDuplicate16x4>>; |
| using _16x64 = |
| DirectionalPredFuncs_SSE4_1<ColStore64_SSE4_1<WriteDuplicate16x4>>; |
| using _32x8 = |
| DirectionalPredFuncs_SSE4_1<ColStore8_SSE4_1<WriteDuplicate32x4>>; |
| using _32x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate32x4>>; |
| using _32x32 = |
| DirectionalPredFuncs_SSE4_1<ColStore32_SSE4_1<WriteDuplicate32x4>>; |
| using _32x64 = |
| DirectionalPredFuncs_SSE4_1<ColStore64_SSE4_1<WriteDuplicate32x4>>; |
| using _64x16 = |
| DirectionalPredFuncs_SSE4_1<ColStore16_SSE4_1<WriteDuplicate64x4>>; |
| using _64x32 = |
| DirectionalPredFuncs_SSE4_1<ColStore32_SSE4_1<WriteDuplicate64x4>>; |
| using _64x64 = |
| DirectionalPredFuncs_SSE4_1<ColStore64_SSE4_1<WriteDuplicate64x4>>; |
| }; |
| |
| void Init10bpp() { |
| Dsp* const dsp = dsp_internal::GetWritableDspTable(10); |
| assert(dsp != nullptr); |
| static_cast<void>(dsp); |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize4x4_IntraPredictorDcTop) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorDcTop] = |
| DcDefs::_4x4::DcTop; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize4x4_IntraPredictorDcLeft) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorDcLeft] = |
| DcDefs::_4x4::DcLeft; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize4x4_IntraPredictorDc) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorDc] = |
| DcDefs::_4x4::Dc; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize4x4_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorHorizontal] = |
| DirDefs::_4x4::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize4x8_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorHorizontal] = |
| DirDefs::_4x8::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize4x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorHorizontal] = |
| DirDefs::_4x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize8x4_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorHorizontal] = |
| DirDefs::_8x4::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize8x8_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorHorizontal] = |
| DirDefs::_8x8::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize8x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorHorizontal] = |
| DirDefs::_8x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize8x32_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorHorizontal] = |
| DirDefs::_8x32::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize16x4_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorHorizontal] = |
| DirDefs::_16x4::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize16x8_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorHorizontal] = |
| DirDefs::_16x8::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize16x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorHorizontal] = |
| DirDefs::_16x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize16x32_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorHorizontal] = |
| DirDefs::_16x32::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize16x64_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorHorizontal] = |
| DirDefs::_16x64::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize32x8_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorHorizontal] = |
| DirDefs::_32x8::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize32x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorHorizontal] = |
| DirDefs::_32x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize32x32_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorHorizontal] = |
| DirDefs::_32x32::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize32x64_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorHorizontal] = |
| DirDefs::_32x64::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize64x16_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorHorizontal] = |
| DirDefs::_64x16::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize64x32_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorHorizontal] = |
| DirDefs::_64x32::Horizontal; |
| #endif |
| #if DSP_ENABLED_10BPP_SSE4_1(TransformSize64x64_IntraPredictorHorizontal) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorHorizontal] = |
| DirDefs::_64x64::Horizontal; |
| #endif |
| } |
| |
| } // namespace |
| } // namespace high_bitdepth |
| #endif // LIBGAV1_MAX_BITDEPTH >= 10 |
| |
| void IntraPredInit_SSE4_1() { |
| low_bitdepth::Init8bpp(); |
| #if LIBGAV1_MAX_BITDEPTH >= 10 |
| high_bitdepth::Init10bpp(); |
| #endif |
| } |
| |
| } // namespace dsp |
| } // namespace libgav1 |
| |
| #else // !LIBGAV1_ENABLE_SSE4_1 |
| namespace libgav1 { |
| namespace dsp { |
| |
| void IntraPredInit_SSE4_1() {} |
| |
| } // namespace dsp |
| } // namespace libgav1 |
| #endif // LIBGAV1_ENABLE_SSE4_1 |