| /****************************************************************************** |
| * |
| * Copyright (C) 2018 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at: |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| ***************************************************************************** |
| * Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore |
| */ |
| |
| /** |
| ******************************************************************************* |
| * @file |
| * ihevc_quant_iquant_ssd_neon_intr.c |
| * |
| * @brief |
| * Contains function definitions for quantization, followed by Inverse |
| * quantization to find transform domain SSD |
| * |
| * @author |
| * 100736 |
| * |
| * @par List of Functions: |
| * - ihevc_quant_iquant_ssd_flat_scale_mat_neon() |
| * - ihevc_q_iq_ssd_flat_scale_mat_var_rnd_fact_neon() |
| * |
| * @remarks |
| * |
| * |
| ******************************************************************************* |
| */ |
| /* System include files */ |
| #include <stdio.h> |
| #include <string.h> |
| #include <stdlib.h> |
| |
| /* User include files */ |
| #include "ihevc_typedefs.h" |
| #include "ihevc_macros.h" |
| #include "ihevc_platform_macros.h" |
| #include "ihevc_defs.h" |
| #include "ihevc_debug.h" |
| #include "ihevc_trans_tables.h" |
| #include "ihevc_quant_iquant_ssd.h" |
| #include "ihevc_func_selector.h" |
| #include "ihevc_trans_macros.h" |
| #include "arm_neon.h" |
| |
| /*****************************************************************************/ |
| /* Function Definitions */ |
| /*****************************************************************************/ |
| |
| WORD32 ihevc_quant_iquant_ssd_flat_scale_mat_neon( |
| WORD16 *pi2_coeffs, |
| WORD16 *pi2_quant_coeff, |
| WORD16 *pi2_q_dst, |
| WORD16 *pi2_iq_dst, |
| WORD32 trans_size, |
| WORD32 qp_div, |
| WORD32 qp_rem, |
| WORD32 q_add, |
| WORD32 *pi4_quant_round_factor_0_1, |
| WORD32 *pi4_quant_round_factor_1_2, |
| WORD32 src_strd, |
| WORD32 dst_q_strd, |
| WORD32 dst_iq_strd, |
| UWORD8 *csbf, |
| WORD32 csbf_strd, |
| WORD32 *zero_col, |
| WORD32 *zero_row, |
| WORD16 *pi2_dequant_coeff, |
| LWORD64 *pi8_cost) |
| { |
| WORD32 i, j; |
| WORD32 log2_size; |
| WORD32 cbf = 0; |
| |
| WORD16 qm = 4; |
| WORD16 bd = 8; |
| WORD32 q_bits, tr, temp; |
| WORD32 block_col = 0; |
| WORD32 block_row = 0; |
| WORD32 temp_zero_col = 0; |
| WORD32 temp_zero_row = 0; |
| |
| WORD32 sh; |
| WORD32 s_iq; |
| WORD32 sh_tmp; |
| |
| // ssd |
| int32x4_t ssd0 = vdupq_n_s32(0); |
| int32x2_t ssd1; |
| WORD32 ssd; |
| // const |
| const int16x4_t zero = vdup_n_s16(0); |
| const int16x4_t zero_d = vdup_n_s16(0); |
| const int16x4_t sq = vdup_n_s16(g_ihevc_quant_scales[qp_rem]); |
| const int16x4_t siq = vdup_n_s16((g_ihevc_iquant_scales_flat_scale[qp_rem])); |
| // src |
| int16x4_t s0, s1, s2, s3; |
| // q-iq |
| int16x4_t q0, q1, q2, q3; |
| int16x4_t iq0, iq1, iq2, iq3; |
| // residue |
| int32x4_t r0, r1, r2, r3; |
| // sign |
| uint16x4_t psgn0, psgn1, psgn2, psgn3; |
| uint16x4_t nsgn0, nsgn1, nsgn2, nsgn3; |
| // abs(src) |
| int16x4_t abs_s0, abs_s1, abs_s2, abs_s3; |
| // q-temp |
| int32x4_t qtmp_0, qtmp_1, qtmp_2, qtmp_3; |
| int16x4_t pq0, pq1, pq2, pq3; |
| int16x4_t nq0, nq1, nq2, nq3; |
| // iq-temp |
| int32x4_t iqtmp_0, iqtmp_1, iqtmp_2, iqtmp_3; |
| |
| int32x4_t add_q; |
| int32x4_t add_iq = vdupq_n_s32(1); |
| int32x4_t sh_iq_1; |
| int32x4_t sh_iq; |
| int32x4_t q_v_bits; |
| |
| (void)pi4_quant_round_factor_0_1; |
| (void)pi4_quant_round_factor_1_2; |
| (void)pi2_dequant_coeff; |
| |
| GETRANGE(log2_size, trans_size); |
| log2_size -= 1; |
| |
| tr = MAX_TR_DYNAMIC_RANGE - bd - log2_size; |
| q_bits = QUANT_SHIFT + qp_div + tr + SCALING_Q_SHIFT - qm - FLAT_RESCALE_MAT_Q_SHIFT; |
| temp = (((WORD32)q_add) << (q_bits - QUANT_ROUND_FACTOR_Q)); |
| |
| q_v_bits = vdupq_n_s32(-q_bits); |
| add_q = vdupq_n_s32(temp); |
| |
| sh = bd + log2_size - 5; |
| |
| sh_tmp = (sh - qp_div - 1); |
| sh_iq_1 = vdupq_n_s32(sh_tmp); |
| add_iq = vshlq_s32(add_iq, sh_iq_1); |
| |
| s_iq = (-(sh - qp_div)); |
| sh_iq = vdupq_n_s32(s_iq); |
| |
| for(i = 0; i < trans_size; i += 4) |
| { |
| for(j = 0; j < trans_size; j += 4) |
| { |
| s0 = vld1_s16(pi2_coeffs + j); |
| s1 = vld1_s16(pi2_coeffs + j + (src_strd)); |
| s2 = vld1_s16(pi2_coeffs + j + (2 * src_strd)); |
| s3 = vld1_s16(pi2_coeffs + j + (3 * src_strd)); |
| |
| /* quantization */ |
| /* sign */ |
| psgn0 = vcge_s16(s0, zero); |
| psgn1 = vcge_s16(s1, zero); |
| psgn2 = vcge_s16(s2, zero); |
| psgn3 = vcge_s16(s3, zero); |
| |
| nsgn0 = vclt_s16(s0, zero); |
| nsgn1 = vclt_s16(s1, zero); |
| nsgn2 = vclt_s16(s2, zero); |
| nsgn3 = vclt_s16(s3, zero); |
| |
| /* |src| */ |
| abs_s0 = vabs_s16(s0); |
| abs_s1 = vabs_s16(s1); |
| abs_s2 = vabs_s16(s2); |
| abs_s3 = vabs_s16(s3); |
| |
| /* tmp = tmp * quant_coeff */ |
| qtmp_0 = vmull_s16(abs_s0, sq); |
| qtmp_1 = vmull_s16(abs_s1, sq); |
| qtmp_2 = vmull_s16(abs_s2, sq); |
| qtmp_3 = vmull_s16(abs_s3, sq); |
| |
| /* tmp += (((WORD32)q_add) << (q_bits - QUANT_ROUND_FACTOR_Q)) */ |
| qtmp_0 = vaddq_s32(qtmp_0, add_q); |
| qtmp_1 = vaddq_s32(qtmp_1, add_q); |
| qtmp_2 = vaddq_s32(qtmp_2, add_q); |
| qtmp_3 = vaddq_s32(qtmp_3, add_q); |
| |
| /* tmp >>= q_bits; */ |
| qtmp_0 = vshlq_s32(qtmp_0, q_v_bits); |
| qtmp_1 = vshlq_s32(qtmp_1, q_v_bits); |
| qtmp_2 = vshlq_s32(qtmp_2, q_v_bits); |
| qtmp_3 = vshlq_s32(qtmp_3, q_v_bits); |
| |
| /* clip */ |
| q0 = vqmovn_s32(qtmp_0); |
| q1 = vqmovn_s32(qtmp_1); |
| q2 = vqmovn_s32(qtmp_2); |
| q3 = vqmovn_s32(qtmp_3); |
| |
| /* restore sign */ |
| pq0 = vand_s16(q0, vreinterpret_s16_u16(psgn0)); |
| pq1 = vand_s16(q1, vreinterpret_s16_u16(psgn1)); |
| pq2 = vand_s16(q2, vreinterpret_s16_u16(psgn2)); |
| pq3 = vand_s16(q3, vreinterpret_s16_u16(psgn3)); |
| |
| nq0 = vand_s16(q0, vreinterpret_s16_u16(nsgn0)); |
| nq1 = vand_s16(q1, vreinterpret_s16_u16(nsgn1)); |
| nq2 = vand_s16(q2, vreinterpret_s16_u16(nsgn2)); |
| nq3 = vand_s16(q3, vreinterpret_s16_u16(nsgn3)); |
| |
| q0 = vsub_s16(pq0, nq0); |
| q1 = vsub_s16(pq1, nq1); |
| q2 = vsub_s16(pq2, nq2); |
| q3 = vsub_s16(pq3, nq3); |
| |
| /* store */ |
| vst1_s16((pi2_q_dst + j), q0); |
| vst1_s16((pi2_q_dst + j + dst_q_strd), q1); |
| vst1_s16((pi2_q_dst + j + (2 * dst_q_strd)), q2); |
| vst1_s16((pi2_q_dst + j + (3 * dst_q_strd)), q3); |
| |
| *(csbf + block_col) = 0; |
| if(vget_lane_s64(vreinterpret_s64_s16(q0), 0) || |
| vget_lane_s64(vreinterpret_s64_s16(q1), 0) || |
| vget_lane_s64(vreinterpret_s64_s16(q2), 0) || |
| vget_lane_s64(vreinterpret_s64_s16(q3), 0)) |
| { |
| *(csbf + block_col) = 1; |
| } |
| |
| if(*(csbf + block_col) == 1) |
| { |
| temp_zero_col |= (0xF << block_col * 4); |
| temp_zero_row |= (0xF << block_row); |
| |
| /* inverse quantization */ |
| iqtmp_0 = vmull_s16(q0, siq); |
| iqtmp_1 = vmull_s16(q1, siq); |
| iqtmp_2 = vmull_s16(q2, siq); |
| iqtmp_3 = vmull_s16(q3, siq); |
| |
| iqtmp_0 = vaddq_s32(iqtmp_0, add_iq); |
| iqtmp_1 = vaddq_s32(iqtmp_1, add_iq); |
| iqtmp_2 = vaddq_s32(iqtmp_2, add_iq); |
| iqtmp_3 = vaddq_s32(iqtmp_3, add_iq); |
| |
| iqtmp_0 = vshlq_s32(iqtmp_0, sh_iq); |
| iqtmp_1 = vshlq_s32(iqtmp_1, sh_iq); |
| iqtmp_2 = vshlq_s32(iqtmp_2, sh_iq); |
| iqtmp_3 = vshlq_s32(iqtmp_3, sh_iq); |
| |
| /* clip */ |
| iq0 = vqmovn_s32(iqtmp_0); |
| iq1 = vqmovn_s32(iqtmp_1); |
| iq2 = vqmovn_s32(iqtmp_2); |
| iq3 = vqmovn_s32(iqtmp_3); |
| |
| /* store */ |
| vst1_s16((pi2_iq_dst + j), iq0); |
| vst1_s16((pi2_iq_dst + j + dst_iq_strd), iq1); |
| vst1_s16((pi2_iq_dst + j + (2 * dst_iq_strd)), iq2); |
| vst1_s16((pi2_iq_dst + j + (3 * dst_iq_strd)), iq3); |
| |
| /* ssd */ |
| /* trans_coeff - inv.quant */ |
| r0 = vsubl_s16(s0, iq0); |
| r1 = vsubl_s16(s1, iq1); |
| r2 = vsubl_s16(s2, iq2); |
| r3 = vsubl_s16(s3, iq3); |
| |
| /* SD */ |
| r0 = vmulq_s32(r0, r0); |
| r1 = vmulq_s32(r1, r1); |
| r2 = vmulq_s32(r2, r2); |
| r3 = vmulq_s32(r3, r3); |
| } |
| else |
| { |
| /* store */ |
| vst1_s16((pi2_iq_dst + j), zero_d); |
| vst1_s16((pi2_iq_dst + j + dst_iq_strd), zero_d); |
| vst1_s16((pi2_iq_dst + j + (2 * dst_iq_strd)), zero_d); |
| vst1_s16((pi2_iq_dst + j + (3 * dst_iq_strd)), zero_d); |
| |
| /* SD */ |
| r0 = vmull_s16(s0, s0); |
| r1 = vmull_s16(s1, s1); |
| r2 = vmull_s16(s2, s2); |
| r3 = vmull_s16(s3, s3); |
| } |
| |
| /* SSD */ |
| r0 = vaddq_s32(r0, r1); |
| r2 = vaddq_s32(r2, r3); |
| |
| r0 = vaddq_s32(r0, r2); |
| |
| /* SSD Accumulation */ |
| ssd0 = vaddq_s32(ssd0, r0); |
| |
| cbf = cbf || (*(csbf + block_col)); // cbf update |
| block_col++; |
| } |
| |
| block_col = 0; |
| block_row += 4; |
| csbf += csbf_strd; |
| |
| pi2_coeffs += 4 * src_strd; |
| pi2_q_dst += 4 * dst_q_strd; |
| pi2_iq_dst += 4 * dst_iq_strd; |
| pi2_quant_coeff += 4 * trans_size; |
| } |
| |
| /* SSD Computation */ |
| ssd1 = vpadd_s32(vget_low_s32(ssd0), vget_high_s32(ssd0)); |
| ssd1 = vpadd_s32(ssd1, ssd1); |
| ssd = vget_lane_s32(ssd1, 0); |
| |
| *zero_col = ~temp_zero_col; //final zero_col storing |
| *zero_row = ~temp_zero_row; //final zero_row storing |
| |
| /* Store the cost */ |
| *pi8_cost = ssd; |
| |
| return cbf; |
| } |
| |
| WORD32 ihevc_q_iq_ssd_flat_scale_mat_var_rnd_fact_neon( |
| WORD16 *pi2_coeffs, |
| WORD16 *pi2_quant_coeff, |
| WORD16 *pi2_q_dst, |
| WORD16 *pi2_iq_dst, |
| WORD32 trans_size, |
| WORD32 qp_div, /* qpscaled / 6 */ |
| WORD32 qp_rem, /* qpscaled % 6 */ |
| WORD32 q_add, |
| WORD32 *pi4_quant_round_factor_0_1, |
| WORD32 *pi4_quant_round_factor_1_2, |
| WORD32 src_strd, |
| WORD32 dst_q_strd, |
| WORD32 dst_iq_strd, |
| UWORD8 *csbf, |
| WORD32 csbf_strd, |
| WORD32 *zero_col, |
| WORD32 *zero_row, |
| WORD16 *pi2_dequant_coeff, |
| LWORD64 *pi8_cost) |
| { |
| WORD32 i, j; |
| WORD32 log2_size; |
| WORD32 cbf = 0; |
| |
| WORD16 qm = 4; |
| WORD16 bd = 8; |
| WORD32 q_bits, tr; |
| WORD32 block_col = 0; |
| WORD32 block_row = 0; |
| WORD32 temp_zero_col = 0; |
| WORD32 temp_zero_row = 0; |
| |
| WORD32 sh; |
| WORD32 s_iq; |
| WORD32 sh_tmp; |
| |
| // ssd |
| int32x4_t ssd0 = vdupq_n_s32(0); |
| int32x2_t ssd1; |
| WORD32 ssd; |
| // const |
| const int16x8_t zero = vdupq_n_s16(0); |
| const int16x4_t zero_d = vdup_n_s16(0); |
| const int16x8_t one = vdupq_n_s16(1); |
| const int16x8_t two = vdupq_n_s16(2); |
| const int16x4_t sq = vdup_n_s16(g_ihevc_quant_scales[qp_rem]); |
| const int16x4_t siq = vdup_n_s16((g_ihevc_iquant_scales_flat_scale[qp_rem])); |
| // src |
| int16x4_t s0, s1, s2, s3; |
| // sign |
| uint16x8_t psgn0, psgn1; |
| uint16x8_t nsgn0, nsgn1; |
| int16x8_t pq0, pq1; |
| int16x8_t nq0, nq1; |
| // abs(src) |
| int16x4_t abs_s0, abs_s1, abs_s2, abs_s3; |
| // q-temp |
| int32x4_t mul_0, mul_1, mul_2, mul_3; |
| int32x4_t q_tmp0, q_tmp1, q_tmp2, q_tmp3; |
| int16x8_t q_00, q_01; |
| int16x8_t q_10, q_11; |
| int16x8_t q_20, q_21; |
| int16x8_t q_30, q_31; |
| // cmp |
| uint16x8_t cmp_00, cmp_01; |
| uint16x8_t cmp_10, cmp_11; |
| uint16x8_t cmp_20, cmp_21; |
| // iq-temp |
| int32x4_t iqtmp_0, iqtmp_1, iqtmp_2, iqtmp_3; |
| int16x4_t iq0, iq1, iq2, iq3; |
| //residue |
| int32x4_t r0, r1, r2, r3; |
| // add_q |
| int32x4_t add_q; |
| int32x4_t add_q0, add_q1, add_q2, add_q3; |
| int32x4_t add_iq = vdupq_n_s32(1); |
| int32x4_t sh_iq_1; |
| int32x4_t sh_iq; |
| int32x4_t q_v_bits; |
| int32x4_t stmp; |
| |
| (void)q_add; |
| (void)pi2_dequant_coeff; |
| GETRANGE(log2_size, trans_size); |
| log2_size -= 1; |
| |
| tr = MAX_TR_DYNAMIC_RANGE - bd - log2_size; |
| q_bits = QUANT_SHIFT + qp_div + tr + SCALING_Q_SHIFT - qm - FLAT_RESCALE_MAT_Q_SHIFT; |
| |
| stmp = vdupq_n_s32(q_bits - QUANT_ROUND_FACTOR_Q); |
| |
| add_q = vdupq_n_s32((1 << QUANT_ROUND_FACTOR_Q) / 2); |
| add_q = vshlq_s32(add_q, stmp); |
| |
| q_v_bits = vdupq_n_s32(-q_bits); |
| |
| sh = bd + log2_size - 5; |
| |
| sh_tmp = (sh - qp_div - 1); |
| sh_iq_1 = vdupq_n_s32(sh_tmp); |
| add_iq = vshlq_s32(add_iq, sh_iq_1); |
| |
| s_iq = (-(sh - qp_div)); |
| sh_iq = vdupq_n_s32(s_iq); |
| |
| for(i = 0; i < trans_size; i += 4) |
| { |
| for(j = 0; j < trans_size; j += 4) |
| { |
| s0 = vld1_s16(pi2_coeffs + j); |
| s1 = vld1_s16(pi2_coeffs + j + (src_strd)); |
| s2 = vld1_s16(pi2_coeffs + j + (2 * src_strd)); |
| s3 = vld1_s16(pi2_coeffs + j + (3 * src_strd)); |
| |
| /* quantization */ |
| /* sign */ |
| psgn0 = vcgeq_s16(vcombine_s16(s0, s1), zero); |
| psgn1 = vcgeq_s16(vcombine_s16(s2, s3), zero); |
| |
| nsgn0 = vcltq_s16(vcombine_s16(s0, s1), zero); |
| nsgn1 = vcltq_s16(vcombine_s16(s2, s3), zero); |
| |
| /* |src| */ |
| abs_s0 = vabs_s16(s0); |
| abs_s1 = vabs_s16(s1); |
| abs_s2 = vabs_s16(s2); |
| abs_s3 = vabs_s16(s3); |
| |
| /* tmp = tmp * quant_coeff */ |
| mul_0 = vmull_s16(abs_s0, sq); |
| mul_1 = vmull_s16(abs_s1, sq); |
| mul_2 = vmull_s16(abs_s2, sq); |
| mul_3 = vmull_s16(abs_s3, sq); |
| |
| /* qadd = 0 */ |
| /* tmp >>= q_bits; */ |
| q_tmp0 = vshlq_s32(mul_0, q_v_bits); |
| q_tmp1 = vshlq_s32(mul_1, q_v_bits); |
| q_tmp2 = vshlq_s32(mul_2, q_v_bits); |
| q_tmp3 = vshlq_s32(mul_3, q_v_bits); |
| |
| /* clip */ |
| q_00 = vcombine_s16(vqmovn_s32(q_tmp0), vqmovn_s32(q_tmp1)); |
| q_01 = vcombine_s16(vqmovn_s32(q_tmp2), vqmovn_s32(q_tmp3)); |
| |
| /* compare qtmp_10, qtmp_20 with 2*/ |
| cmp_00 = vcltq_s16(q_00, two); |
| cmp_01 = vcltq_s16(q_01, two); |
| |
| /* qadd = (1 << QUANT_ROUND_FACTOR_Q)/2) */ |
| /* tmp >>= q_bits; */ |
| q_tmp0 = vaddq_s32(mul_0, add_q); |
| q_tmp1 = vaddq_s32(mul_1, add_q); |
| q_tmp2 = vaddq_s32(mul_2, add_q); |
| q_tmp3 = vaddq_s32(mul_3, add_q); |
| |
| q_tmp0 = vshlq_s32(q_tmp0, q_v_bits); |
| q_tmp1 = vshlq_s32(q_tmp1, q_v_bits); |
| q_tmp2 = vshlq_s32(q_tmp2, q_v_bits); |
| q_tmp3 = vshlq_s32(q_tmp3, q_v_bits); |
| |
| /* clip */ |
| q_10 = vcombine_s16(vqmovn_s32(q_tmp0), vqmovn_s32(q_tmp1)); |
| q_11 = vcombine_s16(vqmovn_s32(q_tmp2), vqmovn_s32(q_tmp3)); |
| |
| if(vget_lane_s64(vreinterpret_s64_u16(vget_low_u16(cmp_00)), 0) || |
| vget_lane_s64(vreinterpret_s64_u16(vget_high_u16(cmp_00)), 0) || |
| vget_lane_s64(vreinterpret_s64_u16(vget_low_u16(cmp_01)), 0) || |
| vget_lane_s64(vreinterpret_s64_u16(vget_high_u16(cmp_01)), 0)) |
| { |
| /* qadd = *pi4_quant_round_factor_1_2 */ |
| /* tmp >>= q_bits; */ |
| add_q0 = vld1q_s32(pi4_quant_round_factor_1_2 + j); |
| add_q1 = vld1q_s32(pi4_quant_round_factor_1_2 + j + (trans_size)); |
| add_q2 = vld1q_s32(pi4_quant_round_factor_1_2 + j + (2 * trans_size)); |
| add_q3 = vld1q_s32(pi4_quant_round_factor_1_2 + j + (3 * trans_size)); |
| |
| add_q0 = vshlq_s32(add_q0, stmp); |
| add_q1 = vshlq_s32(add_q1, stmp); |
| add_q2 = vshlq_s32(add_q2, stmp); |
| add_q3 = vshlq_s32(add_q3, stmp); |
| |
| q_tmp0 = vaddq_s32(mul_0, add_q0); |
| q_tmp1 = vaddq_s32(mul_1, add_q1); |
| q_tmp2 = vaddq_s32(mul_2, add_q2); |
| q_tmp3 = vaddq_s32(mul_3, add_q3); |
| |
| q_tmp0 = vshlq_s32(q_tmp0, q_v_bits); |
| q_tmp1 = vshlq_s32(q_tmp1, q_v_bits); |
| q_tmp2 = vshlq_s32(q_tmp2, q_v_bits); |
| q_tmp3 = vshlq_s32(q_tmp3, q_v_bits); |
| |
| /* clip */ |
| q_20 = vcombine_s16(vqmovn_s32(q_tmp0), vqmovn_s32(q_tmp1)); |
| q_21 = vcombine_s16(vqmovn_s32(q_tmp2), vqmovn_s32(q_tmp3)); |
| |
| /* qadd = *pi4_quant_round_factor_0_1 */ |
| /* tmp >>= q_bits; */ |
| add_q0 = vld1q_s32(pi4_quant_round_factor_0_1 + j); |
| add_q1 = vld1q_s32(pi4_quant_round_factor_0_1 + j + (trans_size)); |
| add_q2 = vld1q_s32(pi4_quant_round_factor_0_1 + j + (2 * trans_size)); |
| add_q3 = vld1q_s32(pi4_quant_round_factor_0_1 + j + (3 * trans_size)); |
| |
| add_q0 = vshlq_s32(add_q0, stmp); |
| add_q1 = vshlq_s32(add_q1, stmp); |
| add_q2 = vshlq_s32(add_q2, stmp); |
| add_q3 = vshlq_s32(add_q3, stmp); |
| |
| q_tmp0 = vaddq_s32(mul_0, add_q0); |
| q_tmp1 = vaddq_s32(mul_1, add_q1); |
| q_tmp2 = vaddq_s32(mul_2, add_q2); |
| q_tmp3 = vaddq_s32(mul_3, add_q3); |
| |
| q_tmp0 = vshlq_s32(q_tmp0, q_v_bits); |
| q_tmp1 = vshlq_s32(q_tmp1, q_v_bits); |
| q_tmp2 = vshlq_s32(q_tmp2, q_v_bits); |
| q_tmp3 = vshlq_s32(q_tmp3, q_v_bits); |
| |
| /* clip */ |
| q_30 = vcombine_s16(vqmovn_s32(q_tmp0), vqmovn_s32(q_tmp1)); |
| q_31 = vcombine_s16(vqmovn_s32(q_tmp2), vqmovn_s32(q_tmp3)); |
| |
| /* compare qtmp_10, qtmp_20 with 1*/ |
| cmp_10 = vcltq_s16(q_00, one); |
| cmp_11 = vcltq_s16(q_01, one); |
| |
| cmp_20 = vbicq_u16(cmp_00, cmp_10); |
| cmp_21 = vbicq_u16(cmp_01, cmp_11); |
| |
| q_10 = vbslq_s16(cmp_10, q_30, q_10); |
| q_11 = vbslq_s16(cmp_11, q_31, q_11); |
| |
| q_10 = vbslq_s16(cmp_20, q_20, q_10); |
| q_11 = vbslq_s16(cmp_21, q_21, q_11); |
| } |
| |
| /* restore sign */ |
| pq0 = vandq_s16(q_10, vreinterpretq_s16_u16(psgn0)); |
| pq1 = vandq_s16(q_11, vreinterpretq_s16_u16(psgn1)); |
| |
| nq0 = vandq_s16(q_10, vreinterpretq_s16_u16(nsgn0)); |
| nq1 = vandq_s16(q_11, vreinterpretq_s16_u16(nsgn1)); |
| |
| q_10 = vsubq_s16(pq0, nq0); |
| q_11 = vsubq_s16(pq1, nq1); |
| |
| /* store */ |
| vst1_s16((pi2_q_dst + j), vget_low_s16(q_10)); |
| vst1_s16((pi2_q_dst + j + dst_q_strd), vget_high_s16(q_10)); |
| vst1_s16((pi2_q_dst + j + (2 * dst_q_strd)), vget_low_s16(q_11)); |
| vst1_s16((pi2_q_dst + j + (3 * dst_q_strd)), vget_high_s16(q_11)); |
| |
| *(csbf + block_col) = 0; |
| if(vget_lane_s64(vreinterpret_s64_s16(vget_low_s16(q_10)), 0) || |
| vget_lane_s64(vreinterpret_s64_s16(vget_high_s16(q_10)), 0) || |
| vget_lane_s64(vreinterpret_s64_s16(vget_low_s16(q_11)), 0) || |
| vget_lane_s64(vreinterpret_s64_s16(vget_high_s16(q_11)), 0)) |
| { |
| *(csbf + block_col) = 1; |
| } |
| |
| if(*(csbf + block_col) == 1) |
| { |
| temp_zero_col |= (0xF << block_col * 4); |
| temp_zero_row |= (0xF << block_row); |
| |
| /* inverse quantization */ |
| iqtmp_0 = vmull_s16(vget_low_s16(q_10), siq); |
| iqtmp_1 = vmull_s16(vget_high_s16(q_10), siq); |
| iqtmp_2 = vmull_s16(vget_low_s16(q_11), siq); |
| iqtmp_3 = vmull_s16(vget_high_s16(q_11), siq); |
| |
| iqtmp_0 = vaddq_s32(iqtmp_0, add_iq); |
| iqtmp_1 = vaddq_s32(iqtmp_1, add_iq); |
| iqtmp_2 = vaddq_s32(iqtmp_2, add_iq); |
| iqtmp_3 = vaddq_s32(iqtmp_3, add_iq); |
| |
| iqtmp_0 = vshlq_s32(iqtmp_0, sh_iq); |
| iqtmp_1 = vshlq_s32(iqtmp_1, sh_iq); |
| iqtmp_2 = vshlq_s32(iqtmp_2, sh_iq); |
| iqtmp_3 = vshlq_s32(iqtmp_3, sh_iq); |
| |
| /* clip */ |
| iq0 = vqmovn_s32(iqtmp_0); |
| iq1 = vqmovn_s32(iqtmp_1); |
| iq2 = vqmovn_s32(iqtmp_2); |
| iq3 = vqmovn_s32(iqtmp_3); |
| |
| /* store */ |
| vst1_s16((pi2_iq_dst + j), iq0); |
| vst1_s16((pi2_iq_dst + j + dst_iq_strd), iq1); |
| vst1_s16((pi2_iq_dst + j + (2 * dst_iq_strd)), iq2); |
| vst1_s16((pi2_iq_dst + j + (3 * dst_iq_strd)), iq3); |
| |
| /* ssd */ |
| /* trans_coeff - inv.quant */ |
| r0 = vsubl_s16(s0, iq0); |
| r1 = vsubl_s16(s1, iq1); |
| r2 = vsubl_s16(s2, iq2); |
| r3 = vsubl_s16(s3, iq3); |
| |
| /* SD */ |
| r0 = vmulq_s32(r0, r0); |
| r1 = vmulq_s32(r1, r1); |
| r2 = vmulq_s32(r2, r2); |
| r3 = vmulq_s32(r3, r3); |
| } |
| else |
| { |
| /* store */ |
| vst1_s16((pi2_iq_dst + j), zero_d); |
| vst1_s16((pi2_iq_dst + j + dst_iq_strd), zero_d); |
| vst1_s16((pi2_iq_dst + j + (2 * dst_iq_strd)), zero_d); |
| vst1_s16((pi2_iq_dst + j + (3 * dst_iq_strd)), zero_d); |
| |
| /* SD */ |
| r0 = vmull_s16(s0, s0); |
| r1 = vmull_s16(s1, s1); |
| r2 = vmull_s16(s2, s2); |
| r3 = vmull_s16(s3, s3); |
| } |
| |
| /* SSD */ |
| r0 = vaddq_s32(r0, r1); |
| r2 = vaddq_s32(r2, r3); |
| |
| r0 = vaddq_s32(r0, r2); |
| |
| /* SSD Accumulation */ |
| ssd0 = vaddq_s32(ssd0, r0); |
| |
| cbf = cbf || (*(csbf + block_col)); // cbf update |
| block_col++; |
| } |
| |
| block_col = 0; |
| block_row += 4; |
| csbf += csbf_strd; |
| |
| pi2_coeffs += 4 * src_strd; |
| pi2_q_dst += 4 * dst_q_strd; |
| pi2_iq_dst += 4 * dst_iq_strd; |
| pi2_quant_coeff += 4 * trans_size; |
| pi4_quant_round_factor_1_2 += 4 * trans_size; |
| pi4_quant_round_factor_0_1 += 4 * trans_size; |
| } |
| |
| /* SSD Computation */ |
| ssd1 = vpadd_s32(vget_low_s32(ssd0), vget_high_s32(ssd0)); |
| ssd1 = vpadd_s32(ssd1, ssd1); |
| ssd = vget_lane_s32(ssd1, 0); |
| |
| *zero_col = ~temp_zero_col; //final zero_col storing |
| *zero_row = ~temp_zero_row; //final zero_row storing |
| |
| /* Store the cost */ |
| *pi8_cost = ssd; |
| |
| return cbf; |
| } |
