| // Copyright 2019 The libgav1 Authors |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "src/dsp/intrapred.h" |
| #include "src/utils/cpu.h" |
| |
| #if LIBGAV1_ENABLE_SSE4_1 |
| |
| #include <xmmintrin.h> |
| |
| #include <cassert> |
| #include <cstddef> |
| #include <cstdint> |
| #include <cstring> // memcpy |
| |
| #include "src/dsp/constants.h" |
| #include "src/dsp/dsp.h" |
| #include "src/dsp/x86/common_sse4.h" |
| #include "src/utils/common.h" |
| |
| namespace libgav1 { |
| namespace dsp { |
| namespace low_bitdepth { |
| namespace { |
| |
| // Note these constants are duplicated from intrapred.cc to allow the compiler |
| // to have visibility of the values. This helps reduce loads and in the |
| // creation of the inverse weights. |
| constexpr uint8_t kSmoothWeights[] = { |
| // block dimension = 4 |
| 255, 149, 85, 64, |
| // block dimension = 8 |
| 255, 197, 146, 105, 73, 50, 37, 32, |
| // block dimension = 16 |
| 255, 225, 196, 170, 145, 123, 102, 84, 68, 54, 43, 33, 26, 20, 17, 16, |
| // block dimension = 32 |
| 255, 240, 225, 210, 196, 182, 169, 157, 145, 133, 122, 111, 101, 92, 83, 74, |
| 66, 59, 52, 45, 39, 34, 29, 25, 21, 17, 14, 12, 10, 9, 8, 8, |
| // block dimension = 64 |
| 255, 248, 240, 233, 225, 218, 210, 203, 196, 189, 182, 176, 169, 163, 156, |
| 150, 144, 138, 133, 127, 121, 116, 111, 106, 101, 96, 91, 86, 82, 77, 73, |
| 69, 65, 61, 57, 54, 50, 47, 44, 41, 38, 35, 32, 29, 27, 25, 22, 20, 18, 16, |
| 15, 13, 12, 10, 9, 8, 7, 6, 6, 5, 5, 4, 4, 4}; |
| |
| template <int y_mask> |
| inline void WriteSmoothHorizontalSum4(void* const dest, const __m128i& left, |
| const __m128i& weights, |
| const __m128i& scaled_top_right, |
| const __m128i& round) { |
| const __m128i left_y = _mm_shuffle_epi32(left, y_mask); |
| const __m128i weighted_left_y = _mm_mullo_epi16(left_y, weights); |
| const __m128i pred_sum = _mm_add_epi32(scaled_top_right, weighted_left_y); |
| // Equivalent to RightShiftWithRounding(pred[x][y], 8). |
| const __m128i pred = _mm_srli_epi32(_mm_add_epi32(pred_sum, round), 8); |
| const __m128i cvtepi32_epi8 = _mm_set1_epi32(0x0C080400); |
| Store4(dest, _mm_shuffle_epi8(pred, cvtepi32_epi8)); |
| } |
| |
| template <int y_mask> |
| inline __m128i SmoothVerticalSum4(const __m128i& top, const __m128i& weights, |
| const __m128i& scaled_bottom_left) { |
| const __m128i weights_y = _mm_shuffle_epi32(weights, y_mask); |
| const __m128i weighted_top_y = _mm_mullo_epi16(top, weights_y); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi32(scaled_bottom_left, y_mask); |
| return _mm_add_epi32(scaled_bottom_left_y, weighted_top_y); |
| } |
| |
| template <int y_mask> |
| inline void WriteSmoothVerticalSum4(uint8_t* dest, const __m128i& top, |
| const __m128i& weights, |
| const __m128i& scaled_bottom_left, |
| const __m128i& round) { |
| __m128i pred_sum = |
| SmoothVerticalSum4<y_mask>(top, weights, scaled_bottom_left); |
| // Equivalent to RightShiftWithRounding(pred[x][y], 8). |
| pred_sum = _mm_srli_epi32(_mm_add_epi32(pred_sum, round), 8); |
| const __m128i cvtepi32_epi8 = _mm_set1_epi32(0x0C080400); |
| Store4(dest, _mm_shuffle_epi8(pred_sum, cvtepi32_epi8)); |
| } |
| |
| // For SMOOTH_H, |pixels| is the repeated left value for the row. For SMOOTH_V, |
| // |pixels| is a segment of the top row or the whole top row, and |weights| is |
| // repeated. |
| inline __m128i SmoothDirectionalSum8(const __m128i& pixels, |
| const __m128i& weights, |
| const __m128i& scaled_corner) { |
| const __m128i weighted_px = _mm_mullo_epi16(pixels, weights); |
| return _mm_add_epi16(scaled_corner, weighted_px); |
| } |
| |
| inline void WriteSmoothDirectionalSum8(uint8_t* dest, const __m128i& pixels, |
| const __m128i& weights, |
| const __m128i& scaled_corner, |
| const __m128i& round) { |
| const __m128i pred_sum = |
| SmoothDirectionalSum8(pixels, weights, scaled_corner); |
| // Equivalent to RightShiftWithRounding(pred[x][y], 8). |
| const __m128i pred = _mm_srli_epi16(_mm_add_epi16(pred_sum, round), 8); |
| StoreLo8(dest, _mm_packus_epi16(pred, pred)); |
| } |
| |
| // For Horizontal, pixels1 and pixels2 are the same repeated value. For |
| // Vertical, weights1 and weights2 are the same, and scaled_corner1 and |
| // scaled_corner2 are the same. |
| inline void WriteSmoothDirectionalSum16(uint8_t* dest, const __m128i& pixels1, |
| const __m128i& pixels2, |
| const __m128i& weights1, |
| const __m128i& weights2, |
| const __m128i& scaled_corner1, |
| const __m128i& scaled_corner2, |
| const __m128i& round) { |
| const __m128i weighted_px1 = _mm_mullo_epi16(pixels1, weights1); |
| const __m128i weighted_px2 = _mm_mullo_epi16(pixels2, weights2); |
| const __m128i pred_sum1 = _mm_add_epi16(scaled_corner1, weighted_px1); |
| const __m128i pred_sum2 = _mm_add_epi16(scaled_corner2, weighted_px2); |
| // Equivalent to RightShiftWithRounding(pred[x][y], 8). |
| const __m128i pred1 = _mm_srli_epi16(_mm_add_epi16(pred_sum1, round), 8); |
| const __m128i pred2 = _mm_srli_epi16(_mm_add_epi16(pred_sum2, round), 8); |
| StoreUnaligned16(dest, _mm_packus_epi16(pred1, pred2)); |
| } |
| |
| template <int y_mask> |
| inline void WriteSmoothPredSum4(uint8_t* const dest, const __m128i& top, |
| const __m128i& left, const __m128i& weights_x, |
| const __m128i& weights_y, |
| const __m128i& scaled_bottom_left, |
| const __m128i& scaled_top_right, |
| const __m128i& round) { |
| const __m128i left_y = _mm_shuffle_epi32(left, y_mask); |
| const __m128i weighted_left_y = _mm_mullo_epi32(left_y, weights_x); |
| const __m128i weight_y = _mm_shuffle_epi32(weights_y, y_mask); |
| const __m128i weighted_top = _mm_mullo_epi32(weight_y, top); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi32(scaled_bottom_left, y_mask); |
| const __m128i col_pred = _mm_add_epi32(scaled_bottom_left_y, weighted_left_y); |
| const __m128i row_pred = _mm_add_epi32(scaled_top_right, weighted_top); |
| const __m128i pred_sum = _mm_add_epi32(row_pred, col_pred); |
| |
| // Equivalent to RightShiftWithRounding(pred[x][y], 9). |
| const __m128i pred = _mm_srli_epi32(_mm_add_epi32(pred_sum, round), 9); |
| |
| const __m128i cvtepi32_epi8 = _mm_set1_epi32(0x0C080400); |
| Store4(dest, _mm_shuffle_epi8(pred, cvtepi32_epi8)); |
| } |
| |
| // pixels[0]: above and below_pred interleave vector |
| // pixels[1]: left vector |
| // pixels[2]: right_pred vector |
| inline void LoadSmoothPixels4(const uint8_t* above, const uint8_t* left, |
| const int height, __m128i* pixels) { |
| if (height == 4) { |
| pixels[1] = Load4(left); |
| } else if (height == 8) { |
| pixels[1] = LoadLo8(left); |
| } else { |
| pixels[1] = LoadUnaligned16(left); |
| } |
| |
| const __m128i bottom_left = _mm_set1_epi16(left[height - 1]); |
| const __m128i top = _mm_cvtepu8_epi16(Load4(above)); |
| pixels[0] = _mm_unpacklo_epi16(top, bottom_left); |
| pixels[2] = _mm_set1_epi16(above[3]); |
| } |
| |
| // weight_h[0]: weight_h vector |
| // weight_h[1]: scale - weight_h vector |
| // weight_h[2]: same as [0], second half for height = 16 only |
| // weight_h[3]: same as [1], second half for height = 16 only |
| // weight_w[0]: weights_w and scale - weights_w interleave vector |
| inline void LoadSmoothWeights4(const uint8_t* weight_array, const int height, |
| __m128i* weight_h, __m128i* weight_w) { |
| const __m128i scale = _mm_set1_epi16(256); |
| const __m128i x_weights = Load4(weight_array); |
| weight_h[0] = _mm_cvtepu8_epi16(x_weights); |
| weight_h[1] = _mm_sub_epi16(scale, weight_h[0]); |
| weight_w[0] = _mm_unpacklo_epi16(weight_h[0], weight_h[1]); |
| |
| if (height == 8) { |
| const __m128i y_weights = LoadLo8(weight_array + 4); |
| weight_h[0] = _mm_cvtepu8_epi16(y_weights); |
| weight_h[1] = _mm_sub_epi16(scale, weight_h[0]); |
| } else if (height == 16) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i y_weights = LoadUnaligned16(weight_array + 12); |
| weight_h[0] = _mm_cvtepu8_epi16(y_weights); |
| weight_h[1] = _mm_sub_epi16(scale, weight_h[0]); |
| weight_h[2] = _mm_unpackhi_epi8(y_weights, zero); |
| weight_h[3] = _mm_sub_epi16(scale, weight_h[2]); |
| } |
| } |
| |
| inline void WriteSmoothPred4x8(const __m128i* pixel, const __m128i* weights_y, |
| const __m128i* weight_x, uint8_t* dst, |
| const ptrdiff_t stride, |
| const bool use_second_half) { |
| const __m128i round = _mm_set1_epi32(256); |
| const __m128i mask_increment = _mm_set1_epi16(0x0202); |
| const __m128i cvtepi32_epi8 = _mm_set1_epi32(0x0C080400); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i left = use_second_half ? _mm_unpackhi_epi8(pixel[1], zero) |
| : _mm_unpacklo_epi8(pixel[1], zero); |
| __m128i y_select = _mm_set1_epi16(0x0100); |
| |
| for (int i = 0; i < 8; ++i) { |
| const __m128i weight_y = _mm_shuffle_epi8(weights_y[0], y_select); |
| const __m128i inverted_weight_y = _mm_shuffle_epi8(weights_y[1], y_select); |
| const __m128i interleaved_weights = |
| _mm_unpacklo_epi16(weight_y, inverted_weight_y); |
| __m128i vertical_pred = _mm_madd_epi16(pixel[0], interleaved_weights); |
| |
| __m128i horizontal_vect = _mm_shuffle_epi8(left, y_select); |
| horizontal_vect = _mm_unpacklo_epi16(horizontal_vect, pixel[2]); |
| __m128i sum = _mm_madd_epi16(horizontal_vect, weight_x[0]); |
| |
| sum = _mm_add_epi32(vertical_pred, sum); |
| sum = _mm_add_epi32(sum, round); |
| sum = _mm_srai_epi32(sum, 9); |
| |
| sum = _mm_shuffle_epi8(sum, cvtepi32_epi8); |
| Store4(dst, sum); |
| dst += stride; |
| |
| y_select = _mm_add_epi16(y_select, mask_increment); |
| } |
| } |
| |
| // The interleaving approach has some overhead that causes it to underperform in |
| // the 4x4 case. |
| void Smooth4x4_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* top_row, const void* left_column) { |
| const __m128i top = _mm_cvtepu8_epi32(Load4(top_row)); |
| const __m128i left = _mm_cvtepu8_epi32(Load4(left_column)); |
| const __m128i weights = _mm_cvtepu8_epi32(Load4(kSmoothWeights)); |
| const __m128i scale = _mm_set1_epi32(256); |
| // Fourth short is top_row[3]. |
| const __m128i top_right = _mm_shuffle_epi32(top, 0xFF); |
| // Fourth short is left_column[3]. |
| const __m128i bottom_left = _mm_shuffle_epi32(left, 0xFF); |
| const __m128i inverted_weights = _mm_sub_epi32(scale, weights); |
| const __m128i scaled_top_right = _mm_mullo_epi16(inverted_weights, top_right); |
| const __m128i scaled_bottom_left = |
| _mm_mullo_epi16(inverted_weights, bottom_left); |
| auto* dst = static_cast<uint8_t*>(dest); |
| // AV1 spec 7.11.2.6 (3) describes the sum: |
| // smoothPred[y][x:x+3] = weighted_top + scaled_right + weighted_left[y] + |
| // scaled_bottom[y] This could be a loop, but for the immediate value in the |
| // shuffles. |
| WriteSmoothPredSum4<0>(dst, top, left, weights, weights, scaled_bottom_left, |
| scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothPredSum4<0x55>(dst, top, left, weights, weights, |
| scaled_bottom_left, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothPredSum4<0xAA>(dst, top, left, weights, weights, |
| scaled_bottom_left, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothPredSum4<0xFF>(dst, top, left, weights, weights, |
| scaled_bottom_left, scaled_top_right, scale); |
| } |
| |
| void Smooth4x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* top_row, const void* left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| __m128i weights_x[1]; |
| __m128i weights_y[2]; |
| LoadSmoothWeights4(kSmoothWeights, 8, weights_y, weights_x); |
| __m128i pixels[3]; |
| LoadSmoothPixels4(top_ptr, left_ptr, 8, pixels); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothPred4x8(pixels, weights_y, weights_x, dst, stride, false); |
| } |
| |
| void Smooth4x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* top_row, const void* left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| __m128i weights_x[1]; |
| __m128i weights_y[4]; |
| LoadSmoothWeights4(kSmoothWeights, 16, weights_y, weights_x); |
| __m128i pixels[3]; |
| LoadSmoothPixels4(top_ptr, left_ptr, 16, pixels); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothPred4x8(pixels, weights_y, weights_x, dst, stride, false); |
| dst += stride << 3; |
| WriteSmoothPred4x8(pixels, &weights_y[2], weights_x, dst, stride, true); |
| } |
| |
| // pixels[0]: above and below_pred interleave vector, first half |
| // pixels[1]: above and below_pred interleave vector, second half |
| // pixels[2]: left vector |
| // pixels[3]: right_pred vector |
| // pixels[4]: above and below_pred interleave vector, first half |
| // pixels[5]: above and below_pred interleave vector, second half |
| // pixels[6]: left vector + 16 |
| // pixels[7]: right_pred vector |
| inline void LoadSmoothPixels8(const uint8_t* above, const uint8_t* left, |
| const int height, __m128i* pixels) { |
| const __m128i bottom_left = _mm_set1_epi16(left[height - 1]); |
| __m128i top_row = _mm_cvtepu8_epi16(LoadLo8(above)); |
| pixels[0] = _mm_unpacklo_epi16(top_row, bottom_left); |
| pixels[1] = _mm_unpackhi_epi16(top_row, bottom_left); |
| |
| pixels[3] = _mm_set1_epi16(above[7]); |
| |
| if (height == 4) { |
| pixels[2] = Load4(left); |
| } else if (height == 8) { |
| pixels[2] = LoadLo8(left); |
| } else if (height == 16) { |
| pixels[2] = LoadUnaligned16(left); |
| } else { |
| pixels[2] = LoadUnaligned16(left); |
| pixels[4] = pixels[0]; |
| pixels[5] = pixels[1]; |
| pixels[6] = LoadUnaligned16(left + 16); |
| pixels[7] = pixels[3]; |
| } |
| } |
| |
| // weight_h[0]: weight_h vector |
| // weight_h[1]: scale - weight_h vector |
| // weight_h[2]: same as [0], offset 8 |
| // weight_h[3]: same as [1], offset 8 |
| // weight_h[4]: same as [0], offset 16 |
| // weight_h[5]: same as [1], offset 16 |
| // weight_h[6]: same as [0], offset 24 |
| // weight_h[7]: same as [1], offset 24 |
| // weight_w[0]: weights_w and scale - weights_w interleave vector, first half |
| // weight_w[1]: weights_w and scale - weights_w interleave vector, second half |
| inline void LoadSmoothWeights8(const uint8_t* weight_array, const int height, |
| __m128i* weight_w, __m128i* weight_h) { |
| const int offset = (height < 8) ? 0 : 4; |
| __m128i loaded_weights = LoadUnaligned16(&weight_array[offset]); |
| weight_h[0] = _mm_cvtepu8_epi16(loaded_weights); |
| const __m128i inverter = _mm_set1_epi16(256); |
| weight_h[1] = _mm_sub_epi16(inverter, weight_h[0]); |
| |
| if (height == 4) { |
| loaded_weights = _mm_srli_si128(loaded_weights, 4); |
| __m128i weights_x = _mm_cvtepu8_epi16(loaded_weights); |
| __m128i inverted_weights_x = _mm_sub_epi16(inverter, weights_x); |
| weight_w[0] = _mm_unpacklo_epi16(weights_x, inverted_weights_x); |
| weight_w[1] = _mm_unpackhi_epi16(weights_x, inverted_weights_x); |
| } else { |
| weight_w[0] = _mm_unpacklo_epi16(weight_h[0], weight_h[1]); |
| weight_w[1] = _mm_unpackhi_epi16(weight_h[0], weight_h[1]); |
| } |
| |
| if (height == 16) { |
| const __m128i zero = _mm_setzero_si128(); |
| loaded_weights = LoadUnaligned16(weight_array + 12); |
| weight_h[0] = _mm_cvtepu8_epi16(loaded_weights); |
| weight_h[1] = _mm_sub_epi16(inverter, weight_h[0]); |
| weight_h[2] = _mm_unpackhi_epi8(loaded_weights, zero); |
| weight_h[3] = _mm_sub_epi16(inverter, weight_h[2]); |
| } else if (height == 32) { |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i weight_lo = LoadUnaligned16(weight_array + 28); |
| weight_h[0] = _mm_cvtepu8_epi16(weight_lo); |
| weight_h[1] = _mm_sub_epi16(inverter, weight_h[0]); |
| weight_h[2] = _mm_unpackhi_epi8(weight_lo, zero); |
| weight_h[3] = _mm_sub_epi16(inverter, weight_h[2]); |
| const __m128i weight_hi = LoadUnaligned16(weight_array + 44); |
| weight_h[4] = _mm_cvtepu8_epi16(weight_hi); |
| weight_h[5] = _mm_sub_epi16(inverter, weight_h[4]); |
| weight_h[6] = _mm_unpackhi_epi8(weight_hi, zero); |
| weight_h[7] = _mm_sub_epi16(inverter, weight_h[6]); |
| } |
| } |
| |
| inline void WriteSmoothPred8xH(const __m128i* pixels, const __m128i* weights_x, |
| const __m128i* weights_y, const int height, |
| uint8_t* dst, const ptrdiff_t stride, |
| const bool use_second_half) { |
| const __m128i round = _mm_set1_epi32(256); |
| const __m128i mask_increment = _mm_set1_epi16(0x0202); |
| const __m128i cvt_epu16_epi8 = _mm_set_epi32(0, 0, 0xe0c0a08, 0x6040200); |
| |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i left = use_second_half ? _mm_unpackhi_epi8(pixels[2], zero) |
| : _mm_unpacklo_epi8(pixels[2], zero); |
| __m128i y_select = _mm_set1_epi16(0x100); |
| |
| for (int i = 0; i < height; ++i) { |
| const __m128i weight_y = _mm_shuffle_epi8(weights_y[0], y_select); |
| const __m128i inverted_weight_y = _mm_shuffle_epi8(weights_y[1], y_select); |
| const __m128i interleaved_weights = |
| _mm_unpacklo_epi16(weight_y, inverted_weight_y); |
| const __m128i vertical_sum0 = |
| _mm_madd_epi16(pixels[0], interleaved_weights); |
| const __m128i vertical_sum1 = |
| _mm_madd_epi16(pixels[1], interleaved_weights); |
| |
| __m128i horizontal_pixels = _mm_shuffle_epi8(left, y_select); |
| horizontal_pixels = _mm_unpacklo_epi16(horizontal_pixels, pixels[3]); |
| const __m128i horizontal_sum0 = |
| _mm_madd_epi16(horizontal_pixels, weights_x[0]); |
| const __m128i horizontal_sum1 = |
| _mm_madd_epi16(horizontal_pixels, weights_x[1]); |
| |
| __m128i sum0 = _mm_add_epi32(vertical_sum0, horizontal_sum0); |
| sum0 = _mm_add_epi32(sum0, round); |
| sum0 = _mm_srai_epi32(sum0, 9); |
| |
| __m128i sum1 = _mm_add_epi32(vertical_sum1, horizontal_sum1); |
| sum1 = _mm_add_epi32(sum1, round); |
| sum1 = _mm_srai_epi32(sum1, 9); |
| |
| sum0 = _mm_packus_epi16(sum0, sum1); |
| sum0 = _mm_shuffle_epi8(sum0, cvt_epu16_epi8); |
| StoreLo8(dst, sum0); |
| dst += stride; |
| |
| y_select = _mm_add_epi16(y_select, mask_increment); |
| } |
| } |
| |
| void Smooth8x4_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* top_row, const void* left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| __m128i pixels[4]; |
| LoadSmoothPixels8(top_ptr, left_ptr, 4, pixels); |
| |
| __m128i weights_x[2], weights_y[2]; |
| LoadSmoothWeights8(kSmoothWeights, 4, weights_x, weights_y); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothPred8xH(pixels, weights_x, weights_y, 4, dst, stride, false); |
| } |
| |
| void Smooth8x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* top_row, const void* left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| |
| __m128i pixels[4]; |
| LoadSmoothPixels8(top_ptr, left_ptr, 8, pixels); |
| |
| __m128i weights_x[2], weights_y[2]; |
| LoadSmoothWeights8(kSmoothWeights, 8, weights_x, weights_y); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothPred8xH(pixels, weights_x, weights_y, 8, dst, stride, false); |
| } |
| |
| void Smooth8x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* top_row, const void* left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| __m128i pixels[4]; |
| LoadSmoothPixels8(top_ptr, left_ptr, 16, pixels); |
| |
| __m128i weights_x[2], weights_y[4]; |
| LoadSmoothWeights8(kSmoothWeights, 16, weights_x, weights_y); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothPred8xH(pixels, weights_x, weights_y, 8, dst, stride, false); |
| dst += stride << 3; |
| WriteSmoothPred8xH(pixels, weights_x, &weights_y[2], 8, dst, stride, true); |
| } |
| |
| void Smooth8x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* top_row, const void* left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| __m128i pixels[8]; |
| LoadSmoothPixels8(top_ptr, left_ptr, 32, pixels); |
| |
| __m128i weights_x[2], weights_y[8]; |
| LoadSmoothWeights8(kSmoothWeights, 32, weights_x, weights_y); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothPred8xH(pixels, weights_x, weights_y, 8, dst, stride, false); |
| dst += stride << 3; |
| WriteSmoothPred8xH(pixels, weights_x, &weights_y[2], 8, dst, stride, true); |
| dst += stride << 3; |
| WriteSmoothPred8xH(&pixels[4], weights_x, &weights_y[4], 8, dst, stride, |
| false); |
| dst += stride << 3; |
| WriteSmoothPred8xH(&pixels[4], weights_x, &weights_y[6], 8, dst, stride, |
| true); |
| } |
| |
| template <int width, int height> |
| void SmoothWxH(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const uint8_t* const sm_weights_h = kSmoothWeights + height - 4; |
| const uint8_t* const sm_weights_w = kSmoothWeights + width - 4; |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i scale_value = _mm_set1_epi16(256); |
| const __m128i bottom_left = _mm_cvtsi32_si128(left_ptr[height - 1]); |
| const __m128i top_right = _mm_set1_epi16(top_ptr[width - 1]); |
| const __m128i round = _mm_set1_epi32(256); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y = 0; y < height; ++y) { |
| const __m128i weights_y = _mm_cvtsi32_si128(sm_weights_h[y]); |
| const __m128i left_y = _mm_cvtsi32_si128(left_ptr[y]); |
| const __m128i scale_m_weights_y = _mm_sub_epi16(scale_value, weights_y); |
| __m128i scaled_bottom_left = |
| _mm_mullo_epi16(scale_m_weights_y, bottom_left); |
| const __m128i weight_left_y = |
| _mm_shuffle_epi32(_mm_unpacklo_epi16(weights_y, left_y), 0); |
| scaled_bottom_left = _mm_add_epi32(scaled_bottom_left, round); |
| scaled_bottom_left = _mm_shuffle_epi32(scaled_bottom_left, 0); |
| for (int x = 0; x < width; x += 8) { |
| const __m128i top_x = LoadLo8(top_ptr + x); |
| const __m128i weights_x = LoadLo8(sm_weights_w + x); |
| const __m128i top_weights_x = _mm_unpacklo_epi8(top_x, weights_x); |
| const __m128i top_weights_x_lo = _mm_cvtepu8_epi16(top_weights_x); |
| const __m128i top_weights_x_hi = _mm_unpackhi_epi8(top_weights_x, zero); |
| |
| // Here opposite weights and pixels are multiplied, where the order of |
| // interleaving is indicated in the names. |
| __m128i pred_lo = _mm_madd_epi16(top_weights_x_lo, weight_left_y); |
| __m128i pred_hi = _mm_madd_epi16(top_weights_x_hi, weight_left_y); |
| |
| // |scaled_bottom_left| is always scaled by the same weight each row, so |
| // we only derive |scaled_top_right| values here. |
| const __m128i inverted_weights_x = |
| _mm_sub_epi16(scale_value, _mm_cvtepu8_epi16(weights_x)); |
| const __m128i scaled_top_right = |
| _mm_mullo_epi16(inverted_weights_x, top_right); |
| const __m128i scaled_top_right_lo = _mm_cvtepu16_epi32(scaled_top_right); |
| const __m128i scaled_top_right_hi = |
| _mm_unpackhi_epi16(scaled_top_right, zero); |
| pred_lo = _mm_add_epi32(pred_lo, scaled_bottom_left); |
| pred_hi = _mm_add_epi32(pred_hi, scaled_bottom_left); |
| pred_lo = _mm_add_epi32(pred_lo, scaled_top_right_lo); |
| pred_hi = _mm_add_epi32(pred_hi, scaled_top_right_hi); |
| |
| // The round value for RightShiftWithRounding was added with |
| // |scaled_bottom_left|. |
| pred_lo = _mm_srli_epi32(pred_lo, 9); |
| pred_hi = _mm_srli_epi32(pred_hi, 9); |
| const __m128i pred = _mm_packus_epi16(pred_lo, pred_hi); |
| StoreLo8(dst + x, _mm_packus_epi16(pred, pred)); |
| } |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal4x4_SSE4_1(void* dest, const ptrdiff_t stride, |
| const void* top_row, const void* left_column) { |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi32(top_ptr[3]); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i left = _mm_cvtepu8_epi32(Load4(left_ptr)); |
| const __m128i weights = _mm_cvtepu8_epi32(Load4(kSmoothWeights)); |
| __m128i scale = _mm_set1_epi32(256); |
| const __m128i inverted_weights = _mm_sub_epi32(scale, weights); |
| const __m128i scaled_top_right = _mm_mullo_epi16(inverted_weights, top_right); |
| scale = _mm_set1_epi32(128); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothHorizontalSum4<0>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0x55>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xAA>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale); |
| } |
| |
| void SmoothHorizontal4x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi32(top[3]); |
| const __m128i weights = _mm_cvtepu8_epi32(Load4(kSmoothWeights)); |
| __m128i scale = _mm_set1_epi32(256); |
| const __m128i inverted_weights = _mm_sub_epi32(scale, weights); |
| const __m128i scaled_top_right = _mm_mullo_epi16(inverted_weights, top_right); |
| scale = _mm_set1_epi32(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| __m128i left = _mm_cvtepu8_epi32(Load4(left_column)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothHorizontalSum4<0>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0x55>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xAA>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| |
| left = _mm_cvtepu8_epi32(Load4(left_ptr + 4)); |
| WriteSmoothHorizontalSum4<0>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0x55>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xAA>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale); |
| } |
| |
| void SmoothHorizontal4x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi32(top[3]); |
| const __m128i weights = _mm_cvtepu8_epi32(Load4(kSmoothWeights)); |
| __m128i scale = _mm_set1_epi32(256); |
| const __m128i inverted_weights = _mm_sub_epi32(scale, weights); |
| const __m128i scaled_top_right = _mm_mullo_epi16(inverted_weights, top_right); |
| scale = _mm_set1_epi32(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| __m128i left = _mm_cvtepu8_epi32(Load4(left_column)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothHorizontalSum4<0>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0x55>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xAA>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| |
| left = _mm_cvtepu8_epi32(Load4(left_ptr + 4)); |
| WriteSmoothHorizontalSum4<0>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0x55>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xAA>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| |
| left = _mm_cvtepu8_epi32(Load4(left_ptr + 8)); |
| WriteSmoothHorizontalSum4<0>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0x55>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xAA>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| |
| left = _mm_cvtepu8_epi32(Load4(left_ptr + 12)); |
| WriteSmoothHorizontalSum4<0>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0x55>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xAA>(dst, left, weights, scaled_top_right, scale); |
| dst += stride; |
| WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale); |
| } |
| |
| void SmoothHorizontal8x4_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[7]); |
| const __m128i left = _mm_cvtepu8_epi16(Load4(left_column)); |
| const __m128i weights = _mm_cvtepu8_epi16(LoadLo8(kSmoothWeights + 4)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_top_right = _mm_mullo_epi16(inverted_weights, top_right); |
| scale = _mm_set1_epi16(128); |
| __m128i y_select = _mm_set1_epi32(0x01000100); |
| __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x03020302); |
| left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x05040504); |
| left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x07060706); |
| left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| } |
| |
| void SmoothHorizontal8x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[7]); |
| const __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| const __m128i weights = _mm_cvtepu8_epi16(LoadLo8(kSmoothWeights + 4)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_top_right = _mm_mullo_epi16(inverted_weights, top_right); |
| scale = _mm_set1_epi16(128); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal8x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[7]); |
| const __m128i weights = _mm_cvtepu8_epi16(LoadLo8(kSmoothWeights + 4)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_top_right = _mm_mullo_epi16(inverted_weights, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal8x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[7]); |
| const __m128i weights = _mm_cvtepu8_epi16(LoadLo8(kSmoothWeights + 4)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_top_right = _mm_mullo_epi16(inverted_weights, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 16)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 24)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal16x4_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[15]); |
| const __m128i left = _mm_cvtepu8_epi16(Load4(left_column)); |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| scale = _mm_set1_epi16(128); |
| __m128i y_mask = _mm_set1_epi32(0x01000100); |
| __m128i left_y = _mm_shuffle_epi8(left, y_mask); |
| auto* dst = static_cast<uint8_t*>(dest); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| y_mask = _mm_set1_epi32(0x03020302); |
| left_y = _mm_shuffle_epi8(left, y_mask); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| y_mask = _mm_set1_epi32(0x05040504); |
| left_y = _mm_shuffle_epi8(left, y_mask); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| y_mask = _mm_set1_epi32(0x07060706); |
| left_y = _mm_shuffle_epi8(left, y_mask); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| } |
| |
| void SmoothHorizontal16x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[15]); |
| const __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| scale = _mm_set1_epi16(128); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal16x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[15]); |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal16x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[15]); |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 16)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 24)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal16x64_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[15]); |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int left_offset = 0; left_offset < 64; left_offset += 8) { |
| const __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + left_offset)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| dst += stride; |
| } |
| } |
| } |
| |
| void SmoothHorizontal32x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[31]); |
| const __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| const __m128i weights_lo = LoadUnaligned16(kSmoothWeights + 28); |
| const __m128i weights_hi = LoadUnaligned16(kSmoothWeights + 44); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lo); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lo, 8)); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_hi); |
| const __m128i weights4 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hi, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| const __m128i scaled_top_right3 = |
| _mm_mullo_epi16(inverted_weights3, top_right); |
| const __m128i scaled_top_right4 = |
| _mm_mullo_epi16(inverted_weights4, top_right); |
| scale = _mm_set1_epi16(128); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal32x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[31]); |
| const __m128i left1 = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| const __m128i weights_lo = LoadUnaligned16(kSmoothWeights + 28); |
| const __m128i weights_hi = LoadUnaligned16(kSmoothWeights + 44); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lo); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lo, 8)); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_hi); |
| const __m128i weights4 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hi, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| const __m128i scaled_top_right3 = |
| _mm_mullo_epi16(inverted_weights3, top_right); |
| const __m128i scaled_top_right4 = |
| _mm_mullo_epi16(inverted_weights4, top_right); |
| scale = _mm_set1_epi16(128); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left1, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| dst += stride; |
| } |
| const __m128i left2 = |
| _mm_cvtepu8_epi16(LoadLo8(static_cast<const uint8_t*>(left_column) + 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left2, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal32x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[31]); |
| const __m128i weights_lo = LoadUnaligned16(kSmoothWeights + 28); |
| const __m128i weights_hi = LoadUnaligned16(kSmoothWeights + 44); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lo); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lo, 8)); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_hi); |
| const __m128i weights4 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hi, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| const __m128i scaled_top_right3 = |
| _mm_mullo_epi16(inverted_weights3, top_right); |
| const __m128i scaled_top_right4 = |
| _mm_mullo_epi16(inverted_weights4, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 16)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| dst += stride; |
| } |
| left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 24)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal32x64_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[31]); |
| const __m128i weights_lo = LoadUnaligned16(kSmoothWeights + 28); |
| const __m128i weights_hi = LoadUnaligned16(kSmoothWeights + 44); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lo); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lo, 8)); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_hi); |
| const __m128i weights4 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hi, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| const __m128i scaled_top_right3 = |
| _mm_mullo_epi16(inverted_weights3, top_right); |
| const __m128i scaled_top_right4 = |
| _mm_mullo_epi16(inverted_weights4, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int left_offset = 0; left_offset < 64; left_offset += 8) { |
| const __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + left_offset)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| dst += stride; |
| } |
| } |
| } |
| |
| void SmoothHorizontal64x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[63]); |
| const __m128i left1 = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| const __m128i weights_lolo = LoadUnaligned16(kSmoothWeights + 60); |
| const __m128i weights_lohi = LoadUnaligned16(kSmoothWeights + 76); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lolo); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lolo, 8)); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_lohi); |
| const __m128i weights4 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lohi, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| const __m128i scaled_top_right3 = |
| _mm_mullo_epi16(inverted_weights3, top_right); |
| const __m128i scaled_top_right4 = |
| _mm_mullo_epi16(inverted_weights4, top_right); |
| const __m128i weights_hilo = LoadUnaligned16(kSmoothWeights + 92); |
| const __m128i weights_hihi = LoadUnaligned16(kSmoothWeights + 108); |
| const __m128i weights5 = _mm_cvtepu8_epi16(weights_hilo); |
| const __m128i weights6 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hilo, 8)); |
| const __m128i weights7 = _mm_cvtepu8_epi16(weights_hihi); |
| const __m128i weights8 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hihi, 8)); |
| const __m128i inverted_weights5 = _mm_sub_epi16(scale, weights5); |
| const __m128i inverted_weights6 = _mm_sub_epi16(scale, weights6); |
| const __m128i inverted_weights7 = _mm_sub_epi16(scale, weights7); |
| const __m128i inverted_weights8 = _mm_sub_epi16(scale, weights8); |
| const __m128i scaled_top_right5 = |
| _mm_mullo_epi16(inverted_weights5, top_right); |
| const __m128i scaled_top_right6 = |
| _mm_mullo_epi16(inverted_weights6, top_right); |
| const __m128i scaled_top_right7 = |
| _mm_mullo_epi16(inverted_weights7, top_right); |
| const __m128i scaled_top_right8 = |
| _mm_mullo_epi16(inverted_weights8, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left1, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| WriteSmoothDirectionalSum16(dst + 32, left_y, left_y, weights5, weights6, |
| scaled_top_right5, scaled_top_right6, scale); |
| WriteSmoothDirectionalSum16(dst + 48, left_y, left_y, weights7, weights8, |
| scaled_top_right7, scaled_top_right8, scale); |
| dst += stride; |
| } |
| const __m128i left2 = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| __m128i left_y = _mm_shuffle_epi8(left2, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| WriteSmoothDirectionalSum16(dst + 32, left_y, left_y, weights5, weights6, |
| scaled_top_right5, scaled_top_right6, scale); |
| WriteSmoothDirectionalSum16(dst + 48, left_y, left_y, weights7, weights8, |
| scaled_top_right7, scaled_top_right8, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal64x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[63]); |
| const __m128i left1 = _mm_cvtepu8_epi16(LoadLo8(left_column)); |
| const __m128i weights_lolo = LoadUnaligned16(kSmoothWeights + 60); |
| const __m128i weights_lohi = LoadUnaligned16(kSmoothWeights + 76); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lolo); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lolo, 8)); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_lohi); |
| const __m128i weights4 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lohi, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| const __m128i scaled_top_right3 = |
| _mm_mullo_epi16(inverted_weights3, top_right); |
| const __m128i scaled_top_right4 = |
| _mm_mullo_epi16(inverted_weights4, top_right); |
| const __m128i weights_hilo = LoadUnaligned16(kSmoothWeights + 92); |
| const __m128i weights_hihi = LoadUnaligned16(kSmoothWeights + 108); |
| const __m128i weights5 = _mm_cvtepu8_epi16(weights_hilo); |
| const __m128i weights6 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hilo, 8)); |
| const __m128i weights7 = _mm_cvtepu8_epi16(weights_hihi); |
| const __m128i weights8 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hihi, 8)); |
| const __m128i inverted_weights5 = _mm_sub_epi16(scale, weights5); |
| const __m128i inverted_weights6 = _mm_sub_epi16(scale, weights6); |
| const __m128i inverted_weights7 = _mm_sub_epi16(scale, weights7); |
| const __m128i inverted_weights8 = _mm_sub_epi16(scale, weights8); |
| const __m128i scaled_top_right5 = |
| _mm_mullo_epi16(inverted_weights5, top_right); |
| const __m128i scaled_top_right6 = |
| _mm_mullo_epi16(inverted_weights6, top_right); |
| const __m128i scaled_top_right7 = |
| _mm_mullo_epi16(inverted_weights7, top_right); |
| const __m128i scaled_top_right8 = |
| _mm_mullo_epi16(inverted_weights8, top_right); |
| scale = _mm_set1_epi16(128); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left1, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| WriteSmoothDirectionalSum16(dst + 32, left_y, left_y, weights5, weights6, |
| scaled_top_right5, scaled_top_right6, scale); |
| WriteSmoothDirectionalSum16(dst + 48, left_y, left_y, weights7, weights8, |
| scaled_top_right7, scaled_top_right8, scale); |
| dst += stride; |
| } |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i left2 = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left2, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| WriteSmoothDirectionalSum16(dst + 32, left_y, left_y, weights5, weights6, |
| scaled_top_right5, scaled_top_right6, scale); |
| WriteSmoothDirectionalSum16(dst + 48, left_y, left_y, weights7, weights8, |
| scaled_top_right7, scaled_top_right8, scale); |
| dst += stride; |
| } |
| const __m128i left3 = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 16)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left3, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| WriteSmoothDirectionalSum16(dst + 32, left_y, left_y, weights5, weights6, |
| scaled_top_right5, scaled_top_right6, scale); |
| WriteSmoothDirectionalSum16(dst + 48, left_y, left_y, weights7, weights8, |
| scaled_top_right7, scaled_top_right8, scale); |
| dst += stride; |
| } |
| const __m128i left4 = _mm_cvtepu8_epi16(LoadLo8(left_ptr + 24)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left4, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| WriteSmoothDirectionalSum16(dst + 32, left_y, left_y, weights5, weights6, |
| scaled_top_right5, scaled_top_right6, scale); |
| WriteSmoothDirectionalSum16(dst + 48, left_y, left_y, weights7, weights8, |
| scaled_top_right7, scaled_top_right8, scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothHorizontal64x64_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const top = static_cast<const uint8_t*>(top_row); |
| const __m128i top_right = _mm_set1_epi16(top[63]); |
| const __m128i weights_lolo = LoadUnaligned16(kSmoothWeights + 60); |
| const __m128i weights_lohi = LoadUnaligned16(kSmoothWeights + 76); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lolo); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lolo, 8)); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_lohi); |
| const __m128i weights4 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_lohi, 8)); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_top_right1 = |
| _mm_mullo_epi16(inverted_weights1, top_right); |
| const __m128i scaled_top_right2 = |
| _mm_mullo_epi16(inverted_weights2, top_right); |
| const __m128i scaled_top_right3 = |
| _mm_mullo_epi16(inverted_weights3, top_right); |
| const __m128i scaled_top_right4 = |
| _mm_mullo_epi16(inverted_weights4, top_right); |
| const __m128i weights_hilo = LoadUnaligned16(kSmoothWeights + 92); |
| const __m128i weights_hihi = LoadUnaligned16(kSmoothWeights + 108); |
| const __m128i weights5 = _mm_cvtepu8_epi16(weights_hilo); |
| const __m128i weights6 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hilo, 8)); |
| const __m128i weights7 = _mm_cvtepu8_epi16(weights_hihi); |
| const __m128i weights8 = _mm_cvtepu8_epi16(_mm_srli_si128(weights_hihi, 8)); |
| const __m128i inverted_weights5 = _mm_sub_epi16(scale, weights5); |
| const __m128i inverted_weights6 = _mm_sub_epi16(scale, weights6); |
| const __m128i inverted_weights7 = _mm_sub_epi16(scale, weights7); |
| const __m128i inverted_weights8 = _mm_sub_epi16(scale, weights8); |
| const __m128i scaled_top_right5 = |
| _mm_mullo_epi16(inverted_weights5, top_right); |
| const __m128i scaled_top_right6 = |
| _mm_mullo_epi16(inverted_weights6, top_right); |
| const __m128i scaled_top_right7 = |
| _mm_mullo_epi16(inverted_weights7, top_right); |
| const __m128i scaled_top_right8 = |
| _mm_mullo_epi16(inverted_weights8, top_right); |
| scale = _mm_set1_epi16(128); |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int left_offset = 0; left_offset < 64; left_offset += 8) { |
| const __m128i left = _mm_cvtepu8_epi16(LoadLo8(left_ptr + left_offset)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i left_y = _mm_shuffle_epi8(left, y_select); |
| WriteSmoothDirectionalSum16(dst, left_y, left_y, weights1, weights2, |
| scaled_top_right1, scaled_top_right2, scale); |
| WriteSmoothDirectionalSum16(dst + 16, left_y, left_y, weights3, weights4, |
| scaled_top_right3, scaled_top_right4, scale); |
| WriteSmoothDirectionalSum16(dst + 32, left_y, left_y, weights5, weights6, |
| scaled_top_right5, scaled_top_right6, scale); |
| WriteSmoothDirectionalSum16(dst + 48, left_y, left_y, weights7, weights8, |
| scaled_top_right7, scaled_top_right8, scale); |
| dst += stride; |
| } |
| } |
| } |
| |
| inline void LoadSmoothVerticalPixels4(const uint8_t* above, const uint8_t* left, |
| const int height, __m128i* pixels) { |
| __m128i top = Load4(above); |
| const __m128i bottom_left = _mm_set1_epi16(left[height - 1]); |
| top = _mm_cvtepu8_epi16(top); |
| pixels[0] = _mm_unpacklo_epi16(top, bottom_left); |
| } |
| |
| // |weight_array| alternates weight vectors from the table with their inverted |
| // (256-w) counterparts. This is precomputed by the compiler when the weights |
| // table is visible to this module. Removing this visibility can cut speed by up |
| // to half in both 4xH and 8xH transforms. |
| inline void LoadSmoothVerticalWeights4(const uint8_t* weight_array, |
| const int height, __m128i* weights) { |
| const __m128i inverter = _mm_set1_epi16(256); |
| |
| if (height == 4) { |
| const __m128i weight = Load4(weight_array); |
| weights[0] = _mm_cvtepu8_epi16(weight); |
| weights[1] = _mm_sub_epi16(inverter, weights[0]); |
| } else if (height == 8) { |
| const __m128i weight = LoadLo8(weight_array + 4); |
| weights[0] = _mm_cvtepu8_epi16(weight); |
| weights[1] = _mm_sub_epi16(inverter, weights[0]); |
| } else { |
| const __m128i weight = LoadUnaligned16(weight_array + 12); |
| const __m128i zero = _mm_setzero_si128(); |
| weights[0] = _mm_cvtepu8_epi16(weight); |
| weights[1] = _mm_sub_epi16(inverter, weights[0]); |
| weights[2] = _mm_unpackhi_epi8(weight, zero); |
| weights[3] = _mm_sub_epi16(inverter, weights[2]); |
| } |
| } |
| |
| inline void WriteSmoothVertical4xH(const __m128i* pixel, const __m128i* weight, |
| const int height, uint8_t* dst, |
| const ptrdiff_t stride) { |
| const __m128i pred_round = _mm_set1_epi32(128); |
| const __m128i mask_increment = _mm_set1_epi16(0x0202); |
| const __m128i cvtepu8_epi32 = _mm_set1_epi32(0xC080400); |
| __m128i y_select = _mm_set1_epi16(0x0100); |
| |
| for (int y = 0; y < height; ++y) { |
| const __m128i weight_y = _mm_shuffle_epi8(weight[0], y_select); |
| const __m128i inverted_weight_y = _mm_shuffle_epi8(weight[1], y_select); |
| const __m128i alternate_weights = |
| _mm_unpacklo_epi16(weight_y, inverted_weight_y); |
| // Here the pixel vector is top_row[0], corner, top_row[1], corner, ... |
| // The madd instruction yields four results of the form: |
| // (top_row[x] * weight[y] + corner * inverted_weight[y]) |
| __m128i sum = _mm_madd_epi16(pixel[0], alternate_weights); |
| sum = _mm_add_epi32(sum, pred_round); |
| sum = _mm_srai_epi32(sum, 8); |
| sum = _mm_shuffle_epi8(sum, cvtepu8_epi32); |
| Store4(dst, sum); |
| dst += stride; |
| y_select = _mm_add_epi16(y_select, mask_increment); |
| } |
| } |
| |
| void SmoothVertical4x4_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left = static_cast<const uint8_t*>(left_column); |
| const auto* const above = static_cast<const uint8_t*>(top_row); |
| auto* dst = static_cast<uint8_t*>(dest); |
| __m128i pixels; |
| LoadSmoothVerticalPixels4(above, left, 4, &pixels); |
| |
| __m128i weights[2]; |
| LoadSmoothVerticalWeights4(kSmoothWeights, 4, weights); |
| |
| WriteSmoothVertical4xH(&pixels, weights, 4, dst, stride); |
| } |
| |
| void SmoothVertical4x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left = static_cast<const uint8_t*>(left_column); |
| const auto* const above = static_cast<const uint8_t*>(top_row); |
| auto* dst = static_cast<uint8_t*>(dest); |
| __m128i pixels; |
| LoadSmoothVerticalPixels4(above, left, 8, &pixels); |
| |
| __m128i weights[2]; |
| LoadSmoothVerticalWeights4(kSmoothWeights, 8, weights); |
| |
| WriteSmoothVertical4xH(&pixels, weights, 8, dst, stride); |
| } |
| |
| void SmoothVertical4x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left = static_cast<const uint8_t*>(left_column); |
| const auto* const above = static_cast<const uint8_t*>(top_row); |
| auto* dst = static_cast<uint8_t*>(dest); |
| __m128i pixels; |
| LoadSmoothVerticalPixels4(above, left, 16, &pixels); |
| |
| __m128i weights[4]; |
| LoadSmoothVerticalWeights4(kSmoothWeights, 16, weights); |
| |
| WriteSmoothVertical4xH(&pixels, weights, 8, dst, stride); |
| dst += stride << 3; |
| WriteSmoothVertical4xH(&pixels, &weights[2], 8, dst, stride); |
| } |
| |
| void SmoothVertical8x4_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[3]); |
| const __m128i weights = _mm_cvtepu8_epi16(Load4(kSmoothWeights)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_bottom_left = |
| _mm_mullo_epi16(inverted_weights, bottom_left); |
| scale = _mm_set1_epi16(128); |
| |
| auto* dst = static_cast<uint8_t*>(dest); |
| __m128i y_select = _mm_set1_epi32(0x01000100); |
| const __m128i top = _mm_cvtepu8_epi16(LoadLo8(top_row)); |
| __m128i weights_y = _mm_shuffle_epi8(weights, y_select); |
| __m128i scaled_bottom_left_y = _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x03020302); |
| weights_y = _mm_shuffle_epi8(weights, y_select); |
| scaled_bottom_left_y = _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x05040504); |
| weights_y = _mm_shuffle_epi8(weights, y_select); |
| scaled_bottom_left_y = _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x07060706); |
| weights_y = _mm_shuffle_epi8(weights, y_select); |
| scaled_bottom_left_y = _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, scale); |
| } |
| |
| void SmoothVertical8x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[7]); |
| const __m128i weights = _mm_cvtepu8_epi16(LoadLo8(kSmoothWeights + 4)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_bottom_left = |
| _mm_mullo_epi16(inverted_weights, bottom_left); |
| scale = _mm_set1_epi16(128); |
| const __m128i top = _mm_cvtepu8_epi16(LoadLo8(top_row)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical8x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[15]); |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights); |
| const __m128i weights2 = _mm_cvtepu8_epi16(_mm_srli_si128(weights, 8)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i scaled_bottom_left1 = |
| _mm_mullo_epi16(inverted_weights1, bottom_left); |
| const __m128i scaled_bottom_left2 = |
| _mm_mullo_epi16(inverted_weights2, bottom_left); |
| scale = _mm_set1_epi16(128); |
| const __m128i top = _mm_cvtepu8_epi16(LoadLo8(top_row)); |
| auto* dst = static_cast<uint8_t*>(dest); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights1, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left1, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights2, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left2, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical8x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[31]); |
| const __m128i weights_lo = LoadUnaligned16(kSmoothWeights + 28); |
| const __m128i weights_hi = LoadUnaligned16(kSmoothWeights + 44); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lo); |
| const __m128i weights2 = _mm_unpackhi_epi8(weights_lo, zero); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_hi); |
| const __m128i weights4 = _mm_unpackhi_epi8(weights_hi, zero); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_bottom_left1 = |
| _mm_mullo_epi16(inverted_weights1, bottom_left); |
| const __m128i scaled_bottom_left2 = |
| _mm_mullo_epi16(inverted_weights2, bottom_left); |
| const __m128i scaled_bottom_left3 = |
| _mm_mullo_epi16(inverted_weights3, bottom_left); |
| const __m128i scaled_bottom_left4 = |
| _mm_mullo_epi16(inverted_weights4, bottom_left); |
| scale = _mm_set1_epi16(128); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i top = _mm_cvtepu8_epi16(LoadLo8(top_row)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights1, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left1, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights2, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left2, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights3, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left3, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights4, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left4, y_select); |
| WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical16x4_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[3]); |
| const __m128i weights = _mm_cvtepu8_epi16(Load4(kSmoothWeights)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_bottom_left = |
| _mm_mullo_epi16(inverted_weights, bottom_left); |
| scale = _mm_set1_epi16(128); |
| const __m128i top = LoadUnaligned16(top_row); |
| const __m128i top_lo = _mm_cvtepu8_epi16(top); |
| const __m128i top_hi = _mm_cvtepu8_epi16(_mm_srli_si128(top, 8)); |
| |
| __m128i y_select = _mm_set1_epi32(0x01000100); |
| __m128i weights_y = _mm_shuffle_epi8(weights, y_select); |
| __m128i scaled_bottom_left_y = _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x03020302); |
| weights_y = _mm_shuffle_epi8(weights, y_select); |
| scaled_bottom_left_y = _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x05040504); |
| weights_y = _mm_shuffle_epi8(weights, y_select); |
| scaled_bottom_left_y = _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| y_select = _mm_set1_epi32(0x07060706); |
| weights_y = _mm_shuffle_epi8(weights, y_select); |
| scaled_bottom_left_y = _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| } |
| |
| void SmoothVertical16x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[7]); |
| const __m128i weights = _mm_cvtepu8_epi16(LoadLo8(kSmoothWeights + 4)); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_bottom_left = |
| _mm_mullo_epi16(inverted_weights, bottom_left); |
| scale = _mm_set1_epi16(128); |
| |
| const __m128i top = LoadUnaligned16(top_row); |
| const __m128i top_lo = _mm_cvtepu8_epi16(top); |
| const __m128i top_hi = _mm_cvtepu8_epi16(_mm_srli_si128(top, 8)); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical16x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[15]); |
| const __m128i zero = _mm_setzero_si128(); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| const __m128i weights_lo = _mm_cvtepu8_epi16(weights); |
| const __m128i weights_hi = _mm_unpackhi_epi8(weights, zero); |
| const __m128i inverted_weights_lo = _mm_sub_epi16(scale, weights_lo); |
| const __m128i inverted_weights_hi = _mm_sub_epi16(scale, weights_hi); |
| const __m128i scaled_bottom_left_lo = |
| _mm_mullo_epi16(inverted_weights_lo, bottom_left); |
| const __m128i scaled_bottom_left_hi = |
| _mm_mullo_epi16(inverted_weights_hi, bottom_left); |
| scale = _mm_set1_epi16(128); |
| |
| const __m128i top = LoadUnaligned16(top_row); |
| const __m128i top_lo = _mm_cvtepu8_epi16(top); |
| const __m128i top_hi = _mm_unpackhi_epi8(top, zero); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights_lo, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left_lo, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights_hi, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left_hi, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical16x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[31]); |
| const __m128i weights_lo = LoadUnaligned16(kSmoothWeights + 28); |
| const __m128i weights_hi = LoadUnaligned16(kSmoothWeights + 44); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lo); |
| const __m128i weights2 = _mm_unpackhi_epi8(weights_lo, zero); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_hi); |
| const __m128i weights4 = _mm_unpackhi_epi8(weights_hi, zero); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_bottom_left1 = |
| _mm_mullo_epi16(inverted_weights1, bottom_left); |
| const __m128i scaled_bottom_left2 = |
| _mm_mullo_epi16(inverted_weights2, bottom_left); |
| const __m128i scaled_bottom_left3 = |
| _mm_mullo_epi16(inverted_weights3, bottom_left); |
| const __m128i scaled_bottom_left4 = |
| _mm_mullo_epi16(inverted_weights4, bottom_left); |
| scale = _mm_set1_epi16(128); |
| |
| const __m128i top = LoadUnaligned16(top_row); |
| const __m128i top_lo = _mm_cvtepu8_epi16(top); |
| const __m128i top_hi = _mm_unpackhi_epi8(top, zero); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights1, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left1, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights2, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left2, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights3, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left3, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights4, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left4, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical16x64_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[63]); |
| const __m128i scale = _mm_set1_epi16(256); |
| const __m128i round = _mm_set1_epi16(128); |
| const __m128i zero = _mm_setzero_si128(); |
| |
| const __m128i top = LoadUnaligned16(top_row); |
| const __m128i top_lo = _mm_cvtepu8_epi16(top); |
| const __m128i top_hi = _mm_unpackhi_epi8(top, zero); |
| const uint8_t* weights_base_ptr = kSmoothWeights + 60; |
| for (int left_offset = 0; left_offset < 64; left_offset += 16) { |
| const __m128i weights = LoadUnaligned16(weights_base_ptr + left_offset); |
| const __m128i weights_lo = _mm_cvtepu8_epi16(weights); |
| const __m128i weights_hi = _mm_unpackhi_epi8(weights, zero); |
| const __m128i inverted_weights_lo = _mm_sub_epi16(scale, weights_lo); |
| const __m128i inverted_weights_hi = _mm_sub_epi16(scale, weights_hi); |
| const __m128i scaled_bottom_left_lo = |
| _mm_mullo_epi16(inverted_weights_lo, bottom_left); |
| const __m128i scaled_bottom_left_hi = |
| _mm_mullo_epi16(inverted_weights_hi, bottom_left); |
| |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights_lo, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left_lo, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights_hi, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left_hi, y_select); |
| WriteSmoothDirectionalSum16(dst, top_lo, top_hi, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| dst += stride; |
| } |
| } |
| } |
| |
| void SmoothVertical32x8_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[7]); |
| const __m128i top_lo = LoadUnaligned16(top_ptr); |
| const __m128i top_hi = LoadUnaligned16(top_ptr + 16); |
| const __m128i top1 = _mm_cvtepu8_epi16(top_lo); |
| const __m128i top2 = _mm_unpackhi_epi8(top_lo, zero); |
| const __m128i top3 = _mm_cvtepu8_epi16(top_hi); |
| const __m128i top4 = _mm_unpackhi_epi8(top_hi, zero); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i weights = _mm_cvtepu8_epi16(LoadLo8(kSmoothWeights + 4)); |
| const __m128i inverted_weights = _mm_sub_epi16(scale, weights); |
| const __m128i scaled_bottom_left = |
| _mm_mullo_epi16(inverted_weights, bottom_left); |
| scale = _mm_set1_epi16(128); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical32x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[15]); |
| const __m128i top_lo = LoadUnaligned16(top_ptr); |
| const __m128i top_hi = LoadUnaligned16(top_ptr + 16); |
| const __m128i top1 = _mm_cvtepu8_epi16(top_lo); |
| const __m128i top2 = _mm_unpackhi_epi8(top_lo, zero); |
| const __m128i top3 = _mm_cvtepu8_epi16(top_hi); |
| const __m128i top4 = _mm_unpackhi_epi8(top_hi, zero); |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights); |
| const __m128i weights2 = _mm_unpackhi_epi8(weights, zero); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i scaled_bottom_left1 = |
| _mm_mullo_epi16(inverted_weights1, bottom_left); |
| const __m128i scaled_bottom_left2 = |
| _mm_mullo_epi16(inverted_weights2, bottom_left); |
| scale = _mm_set1_epi16(128); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights1, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left1, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights2, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left2, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical32x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[31]); |
| const __m128i weights_lo = LoadUnaligned16(kSmoothWeights + 28); |
| const __m128i weights_hi = LoadUnaligned16(kSmoothWeights + 44); |
| const __m128i zero = _mm_setzero_si128(); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i top_lo = LoadUnaligned16(top_ptr); |
| const __m128i top_hi = LoadUnaligned16(top_ptr + 16); |
| const __m128i top1 = _mm_cvtepu8_epi16(top_lo); |
| const __m128i top2 = _mm_unpackhi_epi8(top_lo, zero); |
| const __m128i top3 = _mm_cvtepu8_epi16(top_hi); |
| const __m128i top4 = _mm_unpackhi_epi8(top_hi, zero); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lo); |
| const __m128i weights2 = _mm_unpackhi_epi8(weights_lo, zero); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_hi); |
| const __m128i weights4 = _mm_unpackhi_epi8(weights_hi, zero); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_bottom_left1 = |
| _mm_mullo_epi16(inverted_weights1, bottom_left); |
| const __m128i scaled_bottom_left2 = |
| _mm_mullo_epi16(inverted_weights2, bottom_left); |
| const __m128i scaled_bottom_left3 = |
| _mm_mullo_epi16(inverted_weights3, bottom_left); |
| const __m128i scaled_bottom_left4 = |
| _mm_mullo_epi16(inverted_weights4, bottom_left); |
| scale = _mm_set1_epi16(128); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights1, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left1, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights2, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left2, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights3, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left3, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights4, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left4, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical32x64_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[63]); |
| const __m128i top_lo = LoadUnaligned16(top_ptr); |
| const __m128i top_hi = LoadUnaligned16(top_ptr + 16); |
| const __m128i top1 = _mm_cvtepu8_epi16(top_lo); |
| const __m128i top2 = _mm_unpackhi_epi8(top_lo, zero); |
| const __m128i top3 = _mm_cvtepu8_epi16(top_hi); |
| const __m128i top4 = _mm_unpackhi_epi8(top_hi, zero); |
| const __m128i scale = _mm_set1_epi16(256); |
| const __m128i round = _mm_set1_epi16(128); |
| const uint8_t* weights_base_ptr = kSmoothWeights + 60; |
| for (int left_offset = 0; left_offset < 64; left_offset += 16) { |
| const __m128i weights = LoadUnaligned16(weights_base_ptr + left_offset); |
| const __m128i weights_lo = _mm_cvtepu8_epi16(weights); |
| const __m128i weights_hi = _mm_unpackhi_epi8(weights, zero); |
| const __m128i inverted_weights_lo = _mm_sub_epi16(scale, weights_lo); |
| const __m128i inverted_weights_hi = _mm_sub_epi16(scale, weights_hi); |
| const __m128i scaled_bottom_left_lo = |
| _mm_mullo_epi16(inverted_weights_lo, bottom_left); |
| const __m128i scaled_bottom_left_hi = |
| _mm_mullo_epi16(inverted_weights_hi, bottom_left); |
| |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights_lo, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left_lo, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights_hi, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left_hi, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| dst += stride; |
| } |
| } |
| } |
| |
| void SmoothVertical64x16_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[15]); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i top_lolo = LoadUnaligned16(top_ptr); |
| const __m128i top_lohi = LoadUnaligned16(top_ptr + 16); |
| const __m128i top1 = _mm_cvtepu8_epi16(top_lolo); |
| const __m128i top2 = _mm_unpackhi_epi8(top_lolo, zero); |
| const __m128i top3 = _mm_cvtepu8_epi16(top_lohi); |
| const __m128i top4 = _mm_unpackhi_epi8(top_lohi, zero); |
| |
| const __m128i weights = LoadUnaligned16(kSmoothWeights + 12); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights); |
| const __m128i weights2 = _mm_unpackhi_epi8(weights, zero); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i top_hilo = LoadUnaligned16(top_ptr + 32); |
| const __m128i top_hihi = LoadUnaligned16(top_ptr + 48); |
| const __m128i top5 = _mm_cvtepu8_epi16(top_hilo); |
| const __m128i top6 = _mm_unpackhi_epi8(top_hilo, zero); |
| const __m128i top7 = _mm_cvtepu8_epi16(top_hihi); |
| const __m128i top8 = _mm_unpackhi_epi8(top_hihi, zero); |
| const __m128i scaled_bottom_left1 = |
| _mm_mullo_epi16(inverted_weights1, bottom_left); |
| const __m128i scaled_bottom_left2 = |
| _mm_mullo_epi16(inverted_weights2, bottom_left); |
| scale = _mm_set1_epi16(128); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights1, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left1, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 32, top5, top6, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 48, top7, top8, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights2, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left2, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 32, top5, top6, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 48, top7, top8, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical64x32_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[31]); |
| const __m128i top_lolo = LoadUnaligned16(top_ptr); |
| const __m128i top_lohi = LoadUnaligned16(top_ptr + 16); |
| const __m128i top1 = _mm_cvtepu8_epi16(top_lolo); |
| const __m128i top2 = _mm_unpackhi_epi8(top_lolo, zero); |
| const __m128i top3 = _mm_cvtepu8_epi16(top_lohi); |
| const __m128i top4 = _mm_unpackhi_epi8(top_lohi, zero); |
| const __m128i top_hilo = LoadUnaligned16(top_ptr + 32); |
| const __m128i top_hihi = LoadUnaligned16(top_ptr + 48); |
| const __m128i top5 = _mm_cvtepu8_epi16(top_hilo); |
| const __m128i top6 = _mm_unpackhi_epi8(top_hilo, zero); |
| const __m128i top7 = _mm_cvtepu8_epi16(top_hihi); |
| const __m128i top8 = _mm_unpackhi_epi8(top_hihi, zero); |
| const __m128i weights_lo = LoadUnaligned16(kSmoothWeights + 28); |
| const __m128i weights_hi = LoadUnaligned16(kSmoothWeights + 44); |
| const __m128i weights1 = _mm_cvtepu8_epi16(weights_lo); |
| const __m128i weights2 = _mm_unpackhi_epi8(weights_lo, zero); |
| const __m128i weights3 = _mm_cvtepu8_epi16(weights_hi); |
| const __m128i weights4 = _mm_unpackhi_epi8(weights_hi, zero); |
| __m128i scale = _mm_set1_epi16(256); |
| const __m128i inverted_weights1 = _mm_sub_epi16(scale, weights1); |
| const __m128i inverted_weights2 = _mm_sub_epi16(scale, weights2); |
| const __m128i inverted_weights3 = _mm_sub_epi16(scale, weights3); |
| const __m128i inverted_weights4 = _mm_sub_epi16(scale, weights4); |
| const __m128i scaled_bottom_left1 = |
| _mm_mullo_epi16(inverted_weights1, bottom_left); |
| const __m128i scaled_bottom_left2 = |
| _mm_mullo_epi16(inverted_weights2, bottom_left); |
| const __m128i scaled_bottom_left3 = |
| _mm_mullo_epi16(inverted_weights3, bottom_left); |
| const __m128i scaled_bottom_left4 = |
| _mm_mullo_epi16(inverted_weights4, bottom_left); |
| scale = _mm_set1_epi16(128); |
| |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights1, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left1, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 32, top5, top6, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 48, top7, top8, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights2, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left2, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 32, top5, top6, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 48, top7, top8, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights3, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left3, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 32, top5, top6, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 48, top7, top8, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights4, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left4, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 32, top5, top6, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| WriteSmoothDirectionalSum16(dst + 48, top7, top8, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| scale); |
| dst += stride; |
| } |
| } |
| |
| void SmoothVertical64x64_SSE4_1(void* const dest, const ptrdiff_t stride, |
| const void* const top_row, |
| const void* const left_column) { |
| const auto* const left_ptr = static_cast<const uint8_t*>(left_column); |
| auto* dst = static_cast<uint8_t*>(dest); |
| const auto* const top_ptr = static_cast<const uint8_t*>(top_row); |
| const __m128i zero = _mm_setzero_si128(); |
| const __m128i bottom_left = _mm_set1_epi16(left_ptr[63]); |
| const __m128i top_lolo = LoadUnaligned16(top_ptr); |
| const __m128i top_lohi = LoadUnaligned16(top_ptr + 16); |
| const __m128i top1 = _mm_cvtepu8_epi16(top_lolo); |
| const __m128i top2 = _mm_unpackhi_epi8(top_lolo, zero); |
| const __m128i top3 = _mm_cvtepu8_epi16(top_lohi); |
| const __m128i top4 = _mm_unpackhi_epi8(top_lohi, zero); |
| const __m128i top_hilo = LoadUnaligned16(top_ptr + 32); |
| const __m128i top_hihi = LoadUnaligned16(top_ptr + 48); |
| const __m128i top5 = _mm_cvtepu8_epi16(top_hilo); |
| const __m128i top6 = _mm_unpackhi_epi8(top_hilo, zero); |
| const __m128i top7 = _mm_cvtepu8_epi16(top_hihi); |
| const __m128i top8 = _mm_unpackhi_epi8(top_hihi, zero); |
| const __m128i scale = _mm_set1_epi16(256); |
| const __m128i round = _mm_set1_epi16(128); |
| const uint8_t* weights_base_ptr = kSmoothWeights + 60; |
| for (int left_offset = 0; left_offset < 64; left_offset += 16) { |
| const __m128i weights = LoadUnaligned16(weights_base_ptr + left_offset); |
| const __m128i weights_lo = _mm_cvtepu8_epi16(weights); |
| const __m128i weights_hi = _mm_unpackhi_epi8(weights, zero); |
| const __m128i inverted_weights_lo = _mm_sub_epi16(scale, weights_lo); |
| const __m128i inverted_weights_hi = _mm_sub_epi16(scale, weights_hi); |
| const __m128i scaled_bottom_left_lo = |
| _mm_mullo_epi16(inverted_weights_lo, bottom_left); |
| const __m128i scaled_bottom_left_hi = |
| _mm_mullo_epi16(inverted_weights_hi, bottom_left); |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights_lo, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left_lo, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| WriteSmoothDirectionalSum16(dst + 32, top5, top6, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| WriteSmoothDirectionalSum16(dst + 48, top7, top8, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| dst += stride; |
| } |
| for (int y_mask = 0x01000100; y_mask < 0x0F0E0F0F; y_mask += 0x02020202) { |
| const __m128i y_select = _mm_set1_epi32(y_mask); |
| const __m128i weights_y = _mm_shuffle_epi8(weights_hi, y_select); |
| const __m128i scaled_bottom_left_y = |
| _mm_shuffle_epi8(scaled_bottom_left_hi, y_select); |
| WriteSmoothDirectionalSum16(dst, top1, top2, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| WriteSmoothDirectionalSum16(dst + 16, top3, top4, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| WriteSmoothDirectionalSum16(dst + 32, top5, top6, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| WriteSmoothDirectionalSum16(dst + 48, top7, top8, weights_y, weights_y, |
| scaled_bottom_left_y, scaled_bottom_left_y, |
| round); |
| dst += stride; |
| } |
| } |
| } |
| |
| void Init8bpp() { |
| Dsp* const dsp = dsp_internal::GetWritableDspTable(kBitdepth8); |
| assert(dsp != nullptr); |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorSmooth] = |
| Smooth4x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorSmooth] = |
| Smooth4x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorSmooth] = |
| Smooth4x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorSmooth] = |
| Smooth8x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorSmooth] = |
| Smooth8x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorSmooth] = |
| Smooth8x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorSmooth] = |
| Smooth8x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorSmooth] = |
| SmoothWxH<16, 4>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorSmooth] = |
| SmoothWxH<16, 8>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorSmooth] = |
| SmoothWxH<16, 16>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorSmooth] = |
| SmoothWxH<16, 32>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x64_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorSmooth] = |
| SmoothWxH<16, 64>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorSmooth] = |
| SmoothWxH<32, 8>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorSmooth] = |
| SmoothWxH<32, 16>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorSmooth] = |
| SmoothWxH<32, 32>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x64_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorSmooth] = |
| SmoothWxH<32, 64>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x16_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorSmooth] = |
| SmoothWxH<64, 16>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x32_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorSmooth] = |
| SmoothWxH<64, 32>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x64_IntraPredictorSmooth) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorSmooth] = |
| SmoothWxH<64, 64>; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorSmoothVertical] = |
| SmoothVertical4x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorSmoothVertical] = |
| SmoothVertical4x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorSmoothVertical] = |
| SmoothVertical4x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorSmoothVertical] = |
| SmoothVertical8x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorSmoothVertical] = |
| SmoothVertical8x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorSmoothVertical] = |
| SmoothVertical8x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorSmoothVertical] = |
| SmoothVertical8x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorSmoothVertical] = |
| SmoothVertical16x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorSmoothVertical] = |
| SmoothVertical16x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorSmoothVertical] = |
| SmoothVertical16x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorSmoothVertical] = |
| SmoothVertical16x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x64_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorSmoothVertical] = |
| SmoothVertical16x64_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorSmoothVertical] = |
| SmoothVertical32x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorSmoothVertical] = |
| SmoothVertical32x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorSmoothVertical] = |
| SmoothVertical32x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x64_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorSmoothVertical] = |
| SmoothVertical32x64_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x16_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorSmoothVertical] = |
| SmoothVertical64x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x32_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorSmoothVertical] = |
| SmoothVertical64x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x64_IntraPredictorSmoothVertical) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorSmoothVertical] = |
| SmoothVertical64x64_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize4x4][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal4x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize4x8][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal4x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize4x16][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal4x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize8x4][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal8x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize8x8][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal8x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize8x16][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal8x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize8x32][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal8x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize16x4][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal16x4_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize16x8][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal16x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize16x16][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal16x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize16x32][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal16x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x64_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize16x64][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal16x64_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize32x8][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal32x8_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize32x16][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal32x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize32x32][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal32x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x64_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize32x64][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal32x64_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x16_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize64x16][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal64x16_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x32_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize64x32][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal64x32_SSE4_1; |
| #endif |
| #if DSP_ENABLED_8BPP_SSE4_1(TransformSize64x64_IntraPredictorSmoothHorizontal) |
| dsp->intra_predictors[kTransformSize64x64][kIntraPredictorSmoothHorizontal] = |
| SmoothHorizontal64x64_SSE4_1; |
| #endif |
| } |
| |
| } // namespace |
| } // namespace low_bitdepth |
| |
| void IntraPredSmoothInit_SSE4_1() { low_bitdepth::Init8bpp(); } |
| |
| } // namespace dsp |
| } // namespace libgav1 |
| |
| #else // !LIBGAV1_ENABLE_SSE4_1 |
| |
| namespace libgav1 { |
| namespace dsp { |
| |
| void IntraPredSmoothInit_SSE4_1() {} |
| |
| } // namespace dsp |
| } // namespace libgav1 |
| |
| #endif // LIBGAV1_ENABLE_SSE4_1 |
