| /****************************************************************************** |
| * |
| * Copyright (C) 2018 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at: |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| ***************************************************************************** |
| * Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore |
| */ |
| /** |
| ******************************************************************************* |
| * @file |
| * ihevce_had_compute_neon.c |
| * |
| * @brief |
| * Contains intrinsic definitions of functions for computing had |
| * |
| * @author |
| * Ittiam |
| * |
| * @par List of Functions: |
| * |
| * @remarks |
| * None |
| * |
| ******************************************************************************** |
| */ |
| |
| /*****************************************************************************/ |
| /* File Includes */ |
| /*****************************************************************************/ |
| /* System include files */ |
| #include <string.h> |
| #include <assert.h> |
| #include <arm_neon.h> |
| |
| /* User include files */ |
| #include "ihevc_typedefs.h" |
| #include "itt_video_api.h" |
| #include "ihevc_cmn_utils_neon.h" |
| #include "ihevce_had_satd.h" |
| #include "ihevce_cmn_utils_instr_set_router.h" |
| |
| /*****************************************************************************/ |
| /* Globals */ |
| /*****************************************************************************/ |
| const int16_t gu2_dc_mask[8] = { 0x0000, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff }; |
| |
| /*****************************************************************************/ |
| /* Function Macros */ |
| /*****************************************************************************/ |
| #define RESIDUE(k, is_chroma) \ |
| if(!is_chroma) \ |
| { \ |
| const uint8x8_t s##k = vld1_u8(pu1_src); \ |
| const uint8x8_t p##k = vld1_u8(pu1_pred); \ |
| *r##k = vreinterpretq_s16_u16(vsubl_u8(s##k, p##k)); \ |
| pu1_src += src_strd; \ |
| pu1_pred += pred_strd; \ |
| } \ |
| else \ |
| { \ |
| const uint8x8_t s##k = vld2_u8(pu1_src).val[0]; \ |
| const uint8x8_t p##k = vld2_u8(pu1_pred).val[0]; \ |
| *r##k = vreinterpretq_s16_u16(vsubl_u8(s##k, p##k)); \ |
| pu1_src += src_strd; \ |
| pu1_pred += pred_strd; \ |
| } |
| |
| /*****************************************************************************/ |
| /* Function Definitions */ |
| /*****************************************************************************/ |
| |
| static INLINE void |
| hadamard4x4_2_one_pass(int16x8_t *r0, int16x8_t *r1, int16x8_t *r2, int16x8_t *r3) |
| { |
| const int16x8_t a0 = vaddq_s16(*r0, *r2); |
| const int16x8_t a1 = vaddq_s16(*r1, *r3); |
| const int16x8_t a2 = vsubq_s16(*r0, *r2); |
| const int16x8_t a3 = vsubq_s16(*r1, *r3); |
| |
| *r0 = vaddq_s16(a0, a1); |
| *r1 = vsubq_s16(a0, a1); |
| *r2 = vaddq_s16(a2, a3); |
| *r3 = vsubq_s16(a2, a3); |
| } |
| |
| static INLINE void hadamard4x4_2( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| int16x8_t *r0, |
| int16x8_t *r1, |
| int16x8_t *r2, |
| int16x8_t *r3) |
| { |
| // compute error between src and pred |
| RESIDUE(0, 0); |
| RESIDUE(1, 0); |
| RESIDUE(2, 0); |
| RESIDUE(3, 0); |
| |
| // vertical hadamard tx |
| hadamard4x4_2_one_pass(r0, r1, r2, r3); |
| |
| // transpose |
| transpose_s16_4x4q(r0, r1, r2, r3); |
| |
| // horizontal hadamard tx |
| hadamard4x4_2_one_pass(r0, r1, r2, r3); |
| } |
| |
| static INLINE void hadamard4x4_4( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| int16x8_t *r0, |
| int16x8_t *r1, |
| int16x8_t *r2, |
| int16x8_t *r3, |
| int16x8_t *r4, |
| int16x8_t *r5, |
| int16x8_t *r6, |
| int16x8_t *r7) |
| { |
| // hadamard 4x4_2n |
| hadamard4x4_2(pu1_src, src_strd, pu1_pred, pred_strd, r0, r1, r2, r3); |
| |
| // hadamard 4x4_2n |
| pu1_src += (4 * src_strd); |
| pu1_pred += (4 * pred_strd); |
| hadamard4x4_2(pu1_src, src_strd, pu1_pred, pred_strd, r4, r5, r6, r7); |
| } |
| |
| static INLINE WORD32 hadamard_sad4x4_4(int16x8_t *a, WORD32 *pi4_hsad, WORD32 hsad_stride) |
| { |
| int16x8_t p[8]; |
| int32x4_t b01, b23; |
| int64x2_t c01, c23; |
| int32x2_t d01, d23; |
| |
| // satd |
| p[0] = vabsq_s16(a[0]); |
| p[1] = vabsq_s16(a[1]); |
| p[0] = vaddq_s16(p[0], p[1]); |
| p[2] = vabsq_s16(a[2]); |
| p[3] = vabsq_s16(a[3]); |
| p[2] = vaddq_s16(p[2], p[3]); |
| |
| p[4] = vabsq_s16(a[4]); |
| p[5] = vabsq_s16(a[5]); |
| p[4] = vaddq_s16(p[4], p[5]); |
| p[6] = vabsq_s16(a[6]); |
| p[7] = vabsq_s16(a[7]); |
| p[6] = vaddq_s16(p[6], p[7]); |
| |
| p[0] = vaddq_s16(p[0], p[2]); |
| b01 = vpaddlq_s16(p[0]); |
| c01 = vpaddlq_s32(b01); |
| d01 = vrshrn_n_s64(c01, 2); |
| vst1_s32(pi4_hsad, d01); |
| pi4_hsad += hsad_stride; |
| |
| p[4] = vaddq_s16(p[4], p[6]); |
| b23 = vpaddlq_s16(p[4]); |
| c23 = vpaddlq_s32(b23); |
| d23 = vrshrn_n_s64(c23, 2); |
| vst1_s32(pi4_hsad, d23); |
| |
| d01 = vadd_s32(d01, d23); |
| |
| return (WORD32)(vget_lane_s64(vpaddl_s32(d01), 0)); |
| } |
| |
| static INLINE WORD32 hadamard_sad8x8_using4x4(int16x8_t *a, WORD32 *early_cbf, WORD32 i4_frm_qstep) |
| { |
| int16x8_t p[8]; |
| const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8)); |
| int32x4_t b; |
| int64x2_t c; |
| int64_t satd; |
| WORD32 i; |
| |
| for(i = 0; i < 4; i++) |
| { |
| int16x8_t p0 = vaddq_s16(a[i], a[i + 4]); |
| int16x8_t p1 = vsubq_s16(a[i], a[i + 4]); |
| |
| int16x4_t q0 = vadd_s16(vget_low_s16(p0), vget_high_s16(p0)); |
| int16x4_t q1 = vsub_s16(vget_low_s16(p0), vget_high_s16(p0)); |
| int16x4_t q2 = vadd_s16(vget_low_s16(p1), vget_high_s16(p1)); |
| int16x4_t q3 = vsub_s16(vget_low_s16(p1), vget_high_s16(p1)); |
| |
| a[i] = vcombine_s16(q0, q2); |
| a[i + 4] = vcombine_s16(q1, q3); |
| } |
| |
| #define EARLY_EXIT(k) \ |
| { \ |
| p[k] = vabsq_s16(a[k]); \ |
| if(*early_cbf == 0) \ |
| { \ |
| uint16x8_t cmp; \ |
| cmp = vcgtq_s16(p[k], threshold); \ |
| if(vget_lane_s64(vreinterpret_s64_u16(vget_low_u16(cmp)), 0) || \ |
| vget_lane_s64(vreinterpret_s64_u16(vget_high_u16(cmp)), 0)) \ |
| { \ |
| *early_cbf = 1; \ |
| } \ |
| } \ |
| } |
| // satd |
| EARLY_EXIT(0); |
| EARLY_EXIT(1); |
| p[0] = vaddq_s16(p[0], p[1]); |
| EARLY_EXIT(2); |
| EARLY_EXIT(3); |
| p[2] = vaddq_s16(p[2], p[3]); |
| |
| EARLY_EXIT(4); |
| EARLY_EXIT(5); |
| p[4] = vaddq_s16(p[4], p[5]); |
| EARLY_EXIT(6); |
| EARLY_EXIT(7); |
| #undef EARLY_EXIT |
| p[6] = vaddq_s16(p[6], p[7]); |
| |
| p[0] = vaddq_s16(p[0], p[2]); |
| p[4] = vaddq_s16(p[4], p[6]); |
| p[0] = vaddq_s16(p[0], p[4]); |
| b = vpaddlq_s16(p[0]); |
| c = vpaddlq_s32(b); |
| satd = vget_lane_s64(vadd_s64(vget_low_s64(c), vget_high_s64(c)), 0); |
| |
| return ((satd + 4) >> 3); |
| } |
| |
| static INLINE void hadamard8x8_one_pass( |
| int16x8_t *r0, |
| int16x8_t *r1, |
| int16x8_t *r2, |
| int16x8_t *r3, |
| int16x8_t *r4, |
| int16x8_t *r5, |
| int16x8_t *r6, |
| int16x8_t *r7) |
| { |
| const int16x8_t a0 = vaddq_s16(*r0, *r4); |
| const int16x8_t a4 = vsubq_s16(*r0, *r4); |
| const int16x8_t a1 = vaddq_s16(*r1, *r5); |
| const int16x8_t a5 = vsubq_s16(*r1, *r5); |
| const int16x8_t a2 = vaddq_s16(*r2, *r6); |
| const int16x8_t a6 = vsubq_s16(*r2, *r6); |
| const int16x8_t a3 = vaddq_s16(*r3, *r7); |
| const int16x8_t a7 = vsubq_s16(*r3, *r7); |
| |
| const int16x8_t b0 = vaddq_s16(a0, a2); |
| const int16x8_t b2 = vsubq_s16(a0, a2); |
| const int16x8_t b1 = vaddq_s16(a1, a3); |
| const int16x8_t b3 = vsubq_s16(a1, a3); |
| const int16x8_t b4 = vaddq_s16(a4, a6); |
| const int16x8_t b6 = vsubq_s16(a4, a6); |
| const int16x8_t b5 = vaddq_s16(a5, a7); |
| const int16x8_t b7 = vsubq_s16(a5, a7); |
| |
| *r0 = vaddq_s16(b0, b1); |
| *r1 = vsubq_s16(b0, b1); |
| *r2 = vaddq_s16(b2, b3); |
| *r3 = vsubq_s16(b2, b3); |
| *r4 = vaddq_s16(b4, b5); |
| *r5 = vsubq_s16(b4, b5); |
| *r6 = vaddq_s16(b6, b7); |
| *r7 = vsubq_s16(b6, b7); |
| } |
| |
| static INLINE void hadamard8x8( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| int16x8_t *r0, |
| int16x8_t *r1, |
| int16x8_t *r2, |
| int16x8_t *r3, |
| int16x8_t *r4, |
| int16x8_t *r5, |
| int16x8_t *r6, |
| int16x8_t *r7, |
| WORD32 is_chroma) |
| { |
| // compute error between src and pred |
| RESIDUE(0, is_chroma); |
| RESIDUE(1, is_chroma); |
| RESIDUE(2, is_chroma); |
| RESIDUE(3, is_chroma); |
| RESIDUE(4, is_chroma); |
| RESIDUE(5, is_chroma); |
| RESIDUE(6, is_chroma); |
| RESIDUE(7, is_chroma); |
| |
| // vertical hadamard tx |
| hadamard8x8_one_pass(r0, r1, r2, r3, r4, r5, r6, r7); |
| |
| // transpose |
| transpose_s16_8x8(r0, r1, r2, r3, r4, r5, r6, r7); |
| |
| // horizontal hadamard tx |
| hadamard8x8_one_pass(r0, r1, r2, r3, r4, r5, r6, r7); |
| } |
| |
| static INLINE UWORD32 ihevce_HAD_8x8_8bit_plane_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD32 is_chroma, |
| WORD32 ac_only) |
| { |
| int16x8_t a0, a1, a2, a3, a4, a5, a6, a7; |
| int32x4_t b; |
| int64x2_t c; |
| int64_t satd; |
| |
| // hadamard 8x8 |
| hadamard8x8( |
| pu1_src, src_strd, pu1_pred, pred_strd, &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, is_chroma); |
| |
| if(ac_only) |
| { |
| const int16x8_t mask = vld1q_s16(gu2_dc_mask); |
| a0 = vandq_s16(a0, mask); |
| } |
| |
| // satd |
| a0 = vabsq_s16(a0); |
| a1 = vabsq_s16(a1); |
| a0 = vaddq_s16(a0, a1); |
| a2 = vabsq_s16(a2); |
| a3 = vabsq_s16(a3); |
| a2 = vaddq_s16(a2, a3); |
| |
| a4 = vabsq_s16(a4); |
| a5 = vabsq_s16(a5); |
| a4 = vaddq_s16(a4, a5); |
| a6 = vabsq_s16(a6); |
| a7 = vabsq_s16(a7); |
| a6 = vaddq_s16(a6, a7); |
| |
| a0 = vaddq_s16(a0, a2); |
| a4 = vaddq_s16(a4, a6); |
| a0 = vaddq_s16(a0, a4); |
| b = vpaddlq_s16(a0); |
| c = vpaddlq_s32(b); |
| satd = vget_lane_s64(vadd_s64(vget_low_s64(c), vget_high_s64(c)), 0); |
| |
| return ((satd + 4) >> 3); |
| } |
| |
| static INLINE UWORD32 ihevce_HAD_4x4_8bit_plane_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD32 is_chroma, |
| WORD32 ac_only) |
| { |
| uint8x16_t src_u8, pred_u8; |
| int16x8_t res_01, res_23; |
| int16x4_t h[4]; |
| int16x4_t v[4]; |
| int16x4x2_t trans_4[2]; |
| int16x8_t combined_rows[4]; |
| int32x4x2_t trans_8; |
| int32x4_t sad_32_4[3]; |
| int32x2_t sad_32_2; |
| int64x1_t sad_64_1; |
| int32_t sad; |
| |
| if(!is_chroma) |
| { |
| src_u8 = load_unaligned_u8q(pu1_src, src_strd); |
| pred_u8 = load_unaligned_u8q(pu1_pred, pred_strd); |
| } |
| else |
| { |
| src_u8 = load_unaligned_u8qi(pu1_src, src_strd); |
| pred_u8 = load_unaligned_u8qi(pu1_pred, pred_strd); |
| } |
| res_01 = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(src_u8), vget_low_u8(pred_u8))); |
| res_23 = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(src_u8), vget_high_u8(pred_u8))); |
| |
| h[0] = vadd_s16(vget_low_s16(res_01), vget_high_s16(res_23)); |
| h[1] = vadd_s16(vget_high_s16(res_01), vget_low_s16(res_23)); |
| h[2] = vsub_s16(vget_high_s16(res_01), vget_low_s16(res_23)); |
| h[3] = vsub_s16(vget_low_s16(res_01), vget_high_s16(res_23)); |
| |
| v[0] = vadd_s16(h[0], h[1]); |
| v[1] = vadd_s16(h[3], h[2]); |
| v[2] = vsub_s16(h[0], h[1]); |
| v[3] = vsub_s16(h[3], h[2]); |
| |
| trans_4[0] = vtrn_s16(v[0], v[2]); |
| trans_4[1] = vtrn_s16(v[1], v[3]); |
| |
| combined_rows[0] = vcombine_s16(trans_4[0].val[0], trans_4[1].val[0]); |
| combined_rows[1] = vcombine_s16(trans_4[0].val[1], trans_4[1].val[1]); |
| |
| combined_rows[2] = vaddq_s16(combined_rows[0], combined_rows[1]); |
| combined_rows[3] = vsubq_s16(combined_rows[0], combined_rows[1]); |
| |
| trans_8 = |
| vtrnq_s32(vreinterpretq_s32_s16(combined_rows[2]), vreinterpretq_s32_s16(combined_rows[3])); |
| |
| combined_rows[0] = |
| vaddq_s16(vreinterpretq_s16_s32(trans_8.val[0]), vreinterpretq_s16_s32(trans_8.val[1])); |
| combined_rows[0] = vabsq_s16(combined_rows[0]); |
| combined_rows[1] = |
| vsubq_s16(vreinterpretq_s16_s32(trans_8.val[0]), vreinterpretq_s16_s32(trans_8.val[1])); |
| combined_rows[1] = vabsq_s16(combined_rows[1]); |
| |
| if(ac_only) |
| { |
| const int16x8_t mask = vld1q_s16(gu2_dc_mask); |
| combined_rows[0] = vandq_s16(combined_rows[0], mask); |
| } |
| |
| sad_32_4[0] = vpaddlq_s16(combined_rows[0]); |
| sad_32_4[1] = vpaddlq_s16(combined_rows[1]); |
| sad_32_4[2] = vaddq_s32(sad_32_4[0], sad_32_4[1]); |
| sad_32_2 = vadd_s32(vget_high_s32(sad_32_4[2]), vget_low_s32(sad_32_4[2])); |
| sad_64_1 = vpaddl_s32(sad_32_2); |
| sad = vget_lane_s64(sad_64_1, 0); |
| |
| return ((sad + 2) >> 2); |
| } |
| |
| UWORD32 ihevce_HAD_4x4_8bit_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| (void)pi2_dst; |
| (void)dst_strd; |
| return ihevce_HAD_4x4_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 0); |
| } |
| |
| UWORD32 ihevce_chroma_compute_AC_HAD_4x4_8bit_neon( |
| UWORD8 *pu1_origin, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred_buf, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| (void)pi2_dst; |
| (void)dst_strd; |
| return ihevce_HAD_4x4_8bit_plane_neon(pu1_origin, src_strd, pu1_pred_buf, pred_strd, 1, 1); |
| } |
| |
| UWORD32 ihevce_HAD_8x8_8bit_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| (void)pi2_dst; |
| (void)dst_strd; |
| return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 0); |
| } |
| |
| UWORD32 ihevce_compute_ac_had_8x8_8bit_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| (void)pi2_dst; |
| (void)dst_strd; |
| return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 1); |
| } |
| |
| UWORD32 ihevce_HAD_16x16_8bit_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| int16x8_t b0[8]; |
| int16x8_t b1[8]; |
| int16x8_t b2[8]; |
| int16x8_t b3[8]; |
| uint32x4_t sum = vdupq_n_u32(0); |
| uint64x2_t c; |
| uint64_t satd; |
| WORD32 i; |
| |
| (void)pi2_dst; |
| (void)dst_strd; |
| |
| // hadamard 8x8 - b0 |
| hadamard8x8( |
| pu1_src, |
| src_strd, |
| pu1_pred, |
| pred_strd, |
| &b0[0], |
| &b0[1], |
| &b0[2], |
| &b0[3], |
| &b0[4], |
| &b0[5], |
| &b0[6], |
| &b0[7], |
| 0); |
| // hadamard 8x8 - b1 |
| hadamard8x8( |
| pu1_src + 8, |
| src_strd, |
| pu1_pred + 8, |
| pred_strd, |
| &b1[0], |
| &b1[1], |
| &b1[2], |
| &b1[3], |
| &b1[4], |
| &b1[5], |
| &b1[6], |
| &b1[7], |
| 0); |
| // hadamard 8x8 - b2 |
| hadamard8x8( |
| pu1_src + (8 * src_strd), |
| src_strd, |
| pu1_pred + (8 * pred_strd), |
| pred_strd, |
| &b2[0], |
| &b2[1], |
| &b2[2], |
| &b2[3], |
| &b2[4], |
| &b2[5], |
| &b2[6], |
| &b2[7], |
| 0); |
| // hadamard 8x8 - b3 |
| hadamard8x8( |
| pu1_src + (8 * src_strd) + 8, |
| src_strd, |
| pu1_pred + (8 * pred_strd) + 8, |
| pred_strd, |
| &b3[0], |
| &b3[1], |
| &b3[2], |
| &b3[3], |
| &b3[4], |
| &b3[5], |
| &b3[6], |
| &b3[7], |
| 0); |
| |
| for(i = 0; i < 8; i++) |
| { |
| int16x8_t p0 = vhaddq_s16(b0[i], b1[i]); |
| int16x8_t p1 = vhsubq_s16(b0[i], b1[i]); |
| int16x8_t p2 = vhaddq_s16(b2[i], b3[i]); |
| int16x8_t p3 = vhsubq_s16(b2[i], b3[i]); |
| |
| int16x8_t q0 = vaddq_s16(p0, p2); |
| int16x8_t q1 = vsubq_s16(p0, p2); |
| int16x8_t q2 = vaddq_s16(p1, p3); |
| int16x8_t q3 = vsubq_s16(p1, p3); |
| |
| uint16x8_t r0 = |
| vaddq_u16(vreinterpretq_u16_s16(vabsq_s16(q0)), vreinterpretq_u16_s16(vabsq_s16(q1))); |
| uint16x8_t r1 = |
| vaddq_u16(vreinterpretq_u16_s16(vabsq_s16(q2)), vreinterpretq_u16_s16(vabsq_s16(q3))); |
| |
| uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0)); |
| uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1)); |
| |
| sum = vaddq_u32(sum, s0); |
| sum = vaddq_u32(sum, s1); |
| } |
| |
| c = vpaddlq_u32(sum); |
| satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0); |
| |
| return ((satd + 4) >> 3); |
| } |
| |
| UWORD32 ihevce_chroma_HAD_4x4_8bit_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| (void)pi2_dst; |
| (void)dst_strd; |
| return ihevce_HAD_4x4_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0); |
| } |
| |
| UWORD32 ihevce_chroma_HAD_8x8_8bit_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| (void)pi2_dst; |
| (void)dst_strd; |
| return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0); |
| } |
| |
| UWORD32 ihevce_chroma_HAD_16x16_8bit_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| UWORD32 au4_satd[4]; |
| |
| (void)pi2_dst; |
| (void)dst_strd; |
| au4_satd[0] = ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0); |
| au4_satd[1] = |
| ihevce_HAD_8x8_8bit_plane_neon(pu1_src + 16, src_strd, pu1_pred + 16, pred_strd, 1, 0); |
| au4_satd[2] = ihevce_HAD_8x8_8bit_plane_neon( |
| pu1_src + 8 * src_strd, src_strd, pu1_pred + 8 * pred_strd, pred_strd, 1, 0); |
| au4_satd[3] = ihevce_HAD_8x8_8bit_plane_neon( |
| pu1_src + 8 * src_strd + 16, src_strd, pu1_pred + 8 * pred_strd + 16, pred_strd, 1, 0); |
| |
| return au4_satd[0] + au4_satd[1] + au4_satd[2] + au4_satd[3]; |
| } |
| |
| UWORD32 ihevce_HAD_32x32_8bit_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd) |
| { |
| int16x8_t a[4][4][8]; |
| uint32x4_t sum = vdupq_n_u32(0); |
| WORD32 b8, b16; |
| uint64x2_t c; |
| uint64_t satd; |
| WORD32 i, j; |
| |
| (void)pi2_dst; |
| (void)dst_strd; |
| // hadamard 32x32 |
| for(b16 = 0; b16 < 4; b16++) |
| { |
| UWORD8 *pu1_src_b16 = pu1_src + (b16 >> 1) * (src_strd * 16) + ((b16 & 1) * 16); |
| UWORD8 *pu1_pred_b16 = pu1_pred + (b16 >> 1) * (pred_strd * 16) + ((b16 & 1) * 16); |
| // hadamard 16x16 |
| for(b8 = 0; b8 < 4; b8++) |
| { |
| UWORD8 *pu1_src_b8 = pu1_src_b16 + (b8 >> 1) * (src_strd * 8) + ((b8 & 1) * 8); |
| UWORD8 *pu1_pred_b8 = pu1_pred_b16 + (b8 >> 1) * (pred_strd * 8) + ((b8 & 1) * 8); |
| // hadamard 8x8 |
| hadamard8x8( |
| pu1_src_b8, |
| src_strd, |
| pu1_pred_b8, |
| pred_strd, |
| &a[b16][b8][0], |
| &a[b16][b8][1], |
| &a[b16][b8][2], |
| &a[b16][b8][3], |
| &a[b16][b8][4], |
| &a[b16][b8][5], |
| &a[b16][b8][6], |
| &a[b16][b8][7], |
| 0); |
| } |
| for(i = 0; i < 8; i++) |
| { |
| int16x8_t p0 = vhaddq_s16(a[b16][0][i], a[b16][1][i]); |
| int16x8_t p1 = vhsubq_s16(a[b16][0][i], a[b16][1][i]); |
| int16x8_t p2 = vhaddq_s16(a[b16][2][i], a[b16][3][i]); |
| int16x8_t p3 = vhsubq_s16(a[b16][2][i], a[b16][3][i]); |
| |
| a[b16][0][i] = vaddq_s16(p0, p2); |
| a[b16][1][i] = vsubq_s16(p0, p2); |
| a[b16][2][i] = vaddq_s16(p1, p3); |
| a[b16][3][i] = vsubq_s16(p1, p3); |
| |
| a[b16][0][i] = vshrq_n_s16(a[b16][0][i], 2); |
| a[b16][1][i] = vshrq_n_s16(a[b16][1][i], 2); |
| a[b16][2][i] = vshrq_n_s16(a[b16][2][i], 2); |
| a[b16][3][i] = vshrq_n_s16(a[b16][3][i], 2); |
| } |
| } |
| for(j = 0; j < 4; j++) |
| { |
| for(i = 0; i < 8; i++) |
| { |
| int16x8_t p0 = vaddq_s16(a[0][j][i], a[1][j][i]); |
| int16x8_t p1 = vsubq_s16(a[0][j][i], a[1][j][i]); |
| int16x8_t p2 = vaddq_s16(a[2][j][i], a[3][j][i]); |
| int16x8_t p3 = vsubq_s16(a[2][j][i], a[3][j][i]); |
| |
| int16x8_t q0 = vaddq_s16(p0, p2); |
| int16x8_t q1 = vsubq_s16(p0, p2); |
| int16x8_t q2 = vaddq_s16(p1, p3); |
| int16x8_t q3 = vsubq_s16(p1, p3); |
| |
| uint16x8_t r0 = vaddq_u16( |
| vreinterpretq_u16_s16(vabsq_s16(q0)), vreinterpretq_u16_s16(vabsq_s16(q1))); |
| uint16x8_t r1 = vaddq_u16( |
| vreinterpretq_u16_s16(vabsq_s16(q2)), vreinterpretq_u16_s16(vabsq_s16(q3))); |
| |
| uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0)); |
| uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1)); |
| |
| sum = vaddq_u32(sum, s0); |
| sum = vaddq_u32(sum, s1); |
| } |
| } |
| c = vpaddlq_u32(sum); |
| satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0); |
| |
| return ((satd + 2) >> 2); |
| } |
| |
| WORD32 ihevce_had4_4x4_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst4x4, |
| WORD32 dst_strd, |
| WORD32 *pi4_hsad, |
| WORD32 hsad_stride, |
| WORD32 i4_frm_qstep) |
| { |
| int16x8_t a[8]; |
| |
| (void)pi2_dst4x4; |
| (void)dst_strd; |
| (void)i4_frm_qstep; |
| |
| /* -------- Compute four 4x4 HAD Transforms of 8x8 in one call--------- */ |
| hadamard4x4_4( |
| pu1_src, |
| src_strd, |
| pu1_pred, |
| pred_strd, |
| &a[0], |
| &a[1], |
| &a[2], |
| &a[3], |
| &a[4], |
| &a[5], |
| &a[6], |
| &a[7]); |
| |
| return hadamard_sad4x4_4(a, pi4_hsad, hsad_stride); |
| } |
| |
| WORD32 ihevce_had_8x8_using_4_4x4_r_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd, |
| WORD32 **ppi4_hsad, |
| WORD32 **ppi4_tu_split, |
| WORD32 **ppi4_tu_early_cbf, |
| WORD32 pos_x_y_4x4, |
| WORD32 num_4x4_in_row, |
| WORD32 lambda, |
| WORD32 lambda_q_shift, |
| WORD32 i4_frm_qstep, |
| WORD32 i4_cur_depth, |
| WORD32 i4_max_depth, |
| WORD32 i4_max_tr_size, |
| WORD32 *pi4_tu_split_cost, |
| void *pv_func_sel) |
| { |
| WORD32 pos_x = pos_x_y_4x4 & 0xFFFF; |
| WORD32 pos_y = (pos_x_y_4x4 >> 16) & 0xFFFF; |
| |
| WORD32 *pi4_4x4_hsad; |
| WORD32 *pi4_8x8_hsad; |
| WORD32 *pi4_8x8_tu_split; |
| WORD32 *pi4_8x8_tu_early_cbf; |
| |
| WORD32 cost_child, cost_parent; |
| WORD32 best_cost; |
| WORD32 early_cbf = 0; |
| const UWORD8 u1_cur_tr_size = 8; |
| |
| WORD32 i; |
| |
| int16x8_t a[8]; |
| |
| (void)pv_func_sel; |
| |
| assert(pos_x >= 0); |
| assert(pos_y >= 0); |
| |
| /* Initialize pointers to store 4x4 and 8x8 HAD SATDs */ |
| pi4_4x4_hsad = ppi4_hsad[HAD_4x4] + pos_x + pos_y * num_4x4_in_row; |
| pi4_8x8_hsad = ppi4_hsad[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1); |
| pi4_8x8_tu_split = ppi4_tu_split[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1); |
| pi4_8x8_tu_early_cbf = |
| ppi4_tu_early_cbf[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1); |
| |
| /* -------- Compute four 4x4 HAD Transforms of 8x8 in one call--------- */ |
| hadamard4x4_4( |
| pu1_src, |
| src_strd, |
| pu1_pred, |
| pred_strd, |
| &a[0], |
| &a[1], |
| &a[2], |
| &a[3], |
| &a[4], |
| &a[5], |
| &a[6], |
| &a[7]); |
| |
| /* -------- cost child -------- */ |
| cost_child = hadamard_sad4x4_4(a, pi4_4x4_hsad, num_4x4_in_row); |
| /* 4 CBF Flags, extra 1 becoz of the 0.5 bits per bin is assumed */ |
| cost_child += ((4) * lambda) >> (lambda_q_shift + 1); |
| |
| /* -------- cost parent -------- */ |
| cost_parent = hadamard_sad8x8_using4x4(a, &early_cbf, i4_frm_qstep); |
| for(i = 0; i < 8; i++, pi2_dst += dst_strd) |
| vst1q_s16(pi2_dst, a[i]); |
| |
| if(i4_cur_depth < i4_max_depth) |
| { |
| if((cost_child < cost_parent) || (i4_max_tr_size < u1_cur_tr_size)) |
| { |
| *pi4_tu_split_cost += (4 * lambda) >> (lambda_q_shift + 1); |
| best_cost = cost_child; |
| best_cost <<= 1; |
| best_cost++; |
| pi4_8x8_tu_split[0] = 1; |
| pi4_8x8_hsad[0] = cost_child; |
| } |
| else |
| { |
| best_cost = cost_parent; |
| best_cost <<= 1; |
| pi4_8x8_tu_split[0] = 0; |
| pi4_8x8_hsad[0] = cost_parent; |
| } |
| } |
| else |
| { |
| best_cost = cost_parent; |
| best_cost <<= 1; |
| pi4_8x8_tu_split[0] = 0; |
| pi4_8x8_hsad[0] = cost_parent; |
| } |
| |
| pi4_8x8_tu_early_cbf[0] = early_cbf; |
| |
| /* best cost has tu_split_flag at LSB(Least significant bit) */ |
| return ((best_cost << 1) + early_cbf); |
| } |
| |
| static WORD32 ihevce_compute_16x16HAD_using_8x8_neon( |
| WORD16 *pi2_8x8_had, |
| WORD32 had8_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd, |
| WORD32 i4_frm_qstep, |
| WORD32 *pi4_cbf) |
| { |
| int16x8_t b0[8]; |
| int16x8_t b1[8]; |
| int16x8_t b2[8]; |
| int16x8_t b3[8]; |
| const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8)); |
| uint32x4_t sum = vdupq_n_u32(0); |
| uint64x2_t c; |
| uint64_t satd; |
| WORD32 i; |
| |
| for(i = 0; i < 8; i++, pi2_8x8_had += had8_strd) |
| { |
| b0[i] = vld1q_s16(pi2_8x8_had); |
| b1[i] = vld1q_s16(pi2_8x8_had + 8); |
| } |
| for(i = 0; i < 8; i++, pi2_8x8_had += had8_strd) |
| { |
| b2[i] = vld1q_s16(pi2_8x8_had); |
| b3[i] = vld1q_s16(pi2_8x8_had + 8); |
| } |
| |
| #define EARLY_EXIT(k) \ |
| { \ |
| p##k = vabsq_s16(q##k); \ |
| if(*pi4_cbf == 0) \ |
| { \ |
| uint16x8_t cmp; \ |
| cmp = vcgtq_s16(p##k, threshold); \ |
| if(vget_lane_s64(vreinterpret_s64_u16(vget_low_u16(cmp)), 0) || \ |
| vget_lane_s64(vreinterpret_s64_u16(vget_high_u16(cmp)), 0)) \ |
| { \ |
| *pi4_cbf = 1; \ |
| } \ |
| } \ |
| } |
| for(i = 0; i < 8; i++, pi2_dst += dst_strd) |
| { |
| int16x8_t p0 = vhaddq_s16(b0[i], b1[i]); |
| int16x8_t p1 = vhsubq_s16(b0[i], b1[i]); |
| int16x8_t p2 = vhaddq_s16(b2[i], b3[i]); |
| int16x8_t p3 = vhsubq_s16(b2[i], b3[i]); |
| |
| int16x8_t q0 = vaddq_s16(p0, p2); |
| int16x8_t q1 = vsubq_s16(p0, p2); |
| int16x8_t q2 = vaddq_s16(p1, p3); |
| int16x8_t q3 = vsubq_s16(p1, p3); |
| |
| vst1q_s16(pi2_dst, q0); |
| EARLY_EXIT(0); |
| vst1q_s16(pi2_dst + 8, q1); |
| EARLY_EXIT(1); |
| vst1q_s16(pi2_dst + 8 * dst_strd, q2); |
| EARLY_EXIT(2); |
| vst1q_s16(pi2_dst + 8 * dst_strd + 8, q3); |
| EARLY_EXIT(3); |
| uint16x8_t r0 = vaddq_u16(vreinterpretq_u16_s16(p0), vreinterpretq_u16_s16(p1)); |
| uint16x8_t r1 = vaddq_u16(vreinterpretq_u16_s16(p2), vreinterpretq_u16_s16(p3)); |
| |
| uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0)); |
| uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1)); |
| |
| sum = vaddq_u32(sum, s0); |
| sum = vaddq_u32(sum, s1); |
| } |
| |
| c = vpaddlq_u32(sum); |
| satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0); |
| |
| return ((satd + 4) >> 3); |
| } |
| |
| WORD32 ihevce_had_16x16_r_neon( |
| UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_pred, |
| WORD32 pred_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd, |
| WORD32 **ppi4_hsad, |
| WORD32 **ppi4_tu_split, |
| WORD32 **ppi4_tu_early_cbf, |
| WORD32 pos_x_y_4x4, |
| WORD32 num_4x4_in_row, |
| WORD32 lambda, |
| WORD32 lambda_q_shift, |
| WORD32 i4_frm_qstep, |
| WORD32 i4_cur_depth, |
| WORD32 i4_max_depth, |
| WORD32 i4_max_tr_size, |
| WORD32 *pi4_tu_split_cost, |
| void *pv_func_sel) |
| { |
| WORD16 ai2_8x8_had[256]; |
| |
| WORD32 *pi4_16x16_hsad; |
| WORD32 *pi4_16x16_tu_split; |
| WORD32 *pi4_16x16_tu_early_cbf; |
| |
| WORD32 best_cost, best_cost_tu_split; |
| WORD32 tu_split_flag = 0; |
| WORD32 i4_early_cbf_flag = 0, early_cbf = 0; |
| WORD32 cost_parent, cost_child = 0; |
| |
| const UWORD8 u1_cur_tr_size = 16; |
| |
| WORD32 i; |
| |
| WORD16 *pi2_y0; |
| UWORD8 *src, *pred; |
| WORD32 pos_x_y_4x4_0; |
| |
| WORD32 pos_x = pos_x_y_4x4 & 0xFFFF; |
| WORD32 pos_y = (pos_x_y_4x4 >> 16) & 0xFFFF; |
| |
| assert(pos_x >= 0); |
| assert(pos_y >= 0); |
| |
| /* Initialize pointers to store 16x16 SATDs */ |
| pi4_16x16_hsad = ppi4_hsad[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2); |
| |
| pi4_16x16_tu_split = |
| ppi4_tu_split[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2); |
| |
| pi4_16x16_tu_early_cbf = |
| ppi4_tu_early_cbf[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2); |
| |
| /* -------- Compute four 8x8 HAD Transforms of 16x16 call--------- */ |
| for(i = 0; i < 4; i++) |
| { |
| src = pu1_src + (i & 0x01) * 8 + (i >> 1) * src_strd * 8; |
| pred = pu1_pred + (i & 0x01) * 8 + (i >> 1) * pred_strd * 8; |
| pi2_y0 = ai2_8x8_had + (i & 0x01) * 8 + (i >> 1) * 16 * 8; |
| pos_x_y_4x4_0 = pos_x_y_4x4 + (i & 0x01) * 2 + (i >> 1) * (2 << 16); |
| |
| best_cost_tu_split = ihevce_had_8x8_using_4_4x4_r_neon( |
| src, |
| src_strd, |
| pred, |
| pred_strd, |
| pi2_y0, |
| 16, |
| ppi4_hsad, |
| ppi4_tu_split, |
| ppi4_tu_early_cbf, |
| pos_x_y_4x4_0, |
| num_4x4_in_row, |
| lambda, |
| lambda_q_shift, |
| i4_frm_qstep, |
| i4_cur_depth + 1, |
| i4_max_depth, |
| i4_max_tr_size, |
| pi4_tu_split_cost, |
| pv_func_sel); |
| |
| /* Cost is shifted by two bits for Tu_split_flag and early cbf flag */ |
| best_cost = (best_cost_tu_split >> 2); |
| |
| /* Last but one bit stores the information regarding the TU_Split */ |
| tu_split_flag += (best_cost_tu_split & 0x3) >> 1; |
| |
| /* Last bit stores the information regarding the early_cbf */ |
| i4_early_cbf_flag += (best_cost_tu_split & 0x1); |
| |
| cost_child += best_cost; |
| |
| tu_split_flag <<= 1; |
| i4_early_cbf_flag <<= 1; |
| } |
| |
| /* -------- Compute 16x16 HAD Transform using 8x8 results ------------- */ |
| pi2_y0 = ai2_8x8_had; |
| |
| /* Threshold currently passed as "0" */ |
| cost_parent = ihevce_compute_16x16HAD_using_8x8_neon( |
| pi2_y0, 16, pi2_dst, dst_strd, i4_frm_qstep, &early_cbf); |
| |
| /* 4 TU_Split flags , 4 CBF Flags, extra 1 becoz of the 0.5 bits per bin is assumed */ |
| cost_child += ((4 + 4) * lambda) >> (lambda_q_shift + 1); |
| |
| i4_early_cbf_flag += early_cbf; |
| |
| /* Right now the depth is hard-coded to 4: The depth can be modified from the config file |
| which decides the extent to which TU_REC needs to be done */ |
| if(i4_cur_depth < i4_max_depth) |
| { |
| if((cost_child < cost_parent) || (i4_max_tr_size < u1_cur_tr_size)) |
| { |
| *pi4_tu_split_cost += ((4 + 4) * lambda) >> (lambda_q_shift + 1); |
| tu_split_flag += 1; |
| best_cost = cost_child; |
| } |
| else |
| { |
| tu_split_flag += 0; |
| best_cost = cost_parent; |
| } |
| } |
| else |
| { |
| tu_split_flag += 0; |
| best_cost = cost_parent; |
| } |
| |
| pi4_16x16_hsad[0] = best_cost; |
| pi4_16x16_tu_split[0] = tu_split_flag; |
| pi4_16x16_tu_early_cbf[0] = i4_early_cbf_flag; |
| |
| /*returning two values(best cost & tu_split_flag) as a single value*/ |
| return ((best_cost << 10) + (tu_split_flag << 5) + i4_early_cbf_flag); |
| } |
| |
| UWORD32 ihevce_compute_32x32HAD_using_16x16_neon( |
| WORD16 *pi2_16x16_had, |
| WORD32 had16_strd, |
| WORD16 *pi2_dst, |
| WORD32 dst_strd, |
| WORD32 i4_frm_qstep, |
| WORD32 *pi4_cbf) |
| { |
| int16x8_t a[4][4][8]; |
| uint32x4_t sum = vdupq_n_u32(0); |
| const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8)); |
| WORD32 b8, b16; |
| uint64x2_t c; |
| WORD32 i, j; |
| |
| (void)pi2_dst; |
| (void)dst_strd; |
| |
| for(b16 = 0; b16 < 4; b16++) |
| { |
| WORD16 *pi2_b16 = pi2_16x16_had + (b16 >> 1) * (had16_strd * 16) + ((b16 & 1) * 16); |
| |
| for(b8 = 0; b8 < 4; b8++) |
| { |
| WORD16 *pi2_b8 = pi2_b16 + (b8 >> 1) * (had16_strd * 8) + ((b8 & 1) * 8); |
| |
| for(i = 0; i < 8; i++, pi2_b8 += had16_strd) |
| { |
| a[b16][b8][i] = vld1q_s16(pi2_b8); |
| a[b16][b8][i] = vshrq_n_s16(a[b16][b8][i], 2); |
| } |
| } |
| } |
| |
| for(j = 0; j < 4; j++) |
| { |
| for(i = 0; i < 8; i++) |
| { |
| int16x8_t p0 = vaddq_s16(a[0][j][i], a[1][j][i]); |
| int16x8_t p1 = vsubq_s16(a[0][j][i], a[1][j][i]); |
| int16x8_t p2 = vaddq_s16(a[2][j][i], a[3][j][i]); |
| int16x8_t p3 = vsubq_s16(a[2][j][i], a[3][j][i]); |
| |
| int16x8_t q0 = vaddq_s16(p0, p2); |
| int16x8_t q1 = vsubq_s16(p0, p2); |
| int16x8_t q2 = vaddq_s16(p1, p3); |
| int16x8_t q3 = vsubq_s16(p1, p3); |
| |
| EARLY_EXIT(0); |
| EARLY_EXIT(1); |
| EARLY_EXIT(2); |
| EARLY_EXIT(3); |
| |
| uint16x8_t r0 = vaddq_u16(vreinterpretq_u16_s16(p0), vreinterpretq_u16_s16(p1)); |
| uint16x8_t r1 = vaddq_u16(vreinterpretq_u16_s16(p2), vreinterpretq_u16_s16(p3)); |
| |
| uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0)); |
| uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1)); |
| |
| sum = vaddq_u32(sum, s0); |
| sum = vaddq_u32(sum, s1); |
| } |
| } |
| c = vpaddlq_u32(sum); |
| |
| return vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0); |
| } |
