| /****************************************************************************** |
| * |
| * Copyright (C) 2018 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at: |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| ***************************************************************************** |
| * Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore |
| */ |
| /** |
| ******************************************************************************* |
| * @file |
| * ihevce_hme_utils_neon.c |
| * |
| * @brief |
| * Contains function definitions for hme utils function in neon intrinsic |
| * |
| * |
| * @author |
| * ittian |
| * |
| * @par List of Functions: |
| * - ihevce_get_wt_inp_8x8_neon() |
| * - ihevce_get_wt_inp_ctb_neon() |
| * @remarks |
| * None |
| * |
| ******************************************************************************* |
| */ |
| /*****************************************************************************/ |
| /* File Includes */ |
| /*****************************************************************************/ |
| /* System include files */ |
| #include <stdio.h> |
| #include <string.h> |
| #include <assert.h> |
| #include <arm_neon.h> |
| |
| /* User include files */ |
| #include "ihevc_typedefs.h" |
| #include "itt_video_api.h" |
| #include "ihevc_cmn_utils_neon.h" |
| #include "ihevc_chroma_itrans_recon.h" |
| #include "ihevc_chroma_intra_pred.h" |
| #include "ihevc_debug.h" |
| #include "ihevc_deblk.h" |
| #include "ihevc_defs.h" |
| #include "ihevc_itrans_recon.h" |
| #include "ihevc_intra_pred.h" |
| #include "ihevc_inter_pred.h" |
| #include "ihevc_macros.h" |
| #include "ihevc_mem_fns.h" |
| #include "ihevc_padding.h" |
| #include "ihevc_quant_iquant_ssd.h" |
| #include "ihevc_resi_trans.h" |
| #include "ihevc_sao.h" |
| #include "ihevc_structs.h" |
| #include "ihevc_weighted_pred.h" |
| |
| #include "rc_cntrl_param.h" |
| #include "rc_frame_info_collector.h" |
| #include "rc_look_ahead_params.h" |
| |
| #include "ihevce_api.h" |
| #include "ihevce_defs.h" |
| #include "ihevce_lap_enc_structs.h" |
| #include "ihevce_multi_thrd_structs.h" |
| #include "ihevce_function_selector.h" |
| #include "ihevce_me_common_defs.h" |
| #include "ihevce_enc_structs.h" |
| #include "ihevce_had_satd.h" |
| #include "ihevce_ipe_instr_set_router.h" |
| #include "ihevce_global_tables.h" |
| |
| #include "hme_datatype.h" |
| #include "hme_interface.h" |
| #include "hme_common_defs.h" |
| #include "hme_defs.h" |
| #include "ihevce_me_instr_set_router.h" |
| #include "hme_globals.h" |
| #include "hme_utils.h" |
| #include "hme_coarse.h" |
| #include "hme_refine.h" |
| |
| /*****************************************************************************/ |
| /* Constant Macros */ |
| /*****************************************************************************/ |
| #define IHEVCE_WT_PRED_SHIFT 15 |
| |
| /*****************************************************************************/ |
| /* Function Definitions */ |
| /*****************************************************************************/ |
| |
| static INLINE void ihevce_get_wt_inp_4x8_neon( |
| const UWORD8 *pu1_src, |
| UWORD8 *pu1_dst, |
| wgt_pred_ctxt_t *ps_wt_inp_prms, |
| WORD32 u1_num_valid_refs, |
| WORD32 *ai4_wt_refs, |
| WORD32 src_stride, |
| WORD32 dst_stride) |
| { |
| S32 inv_wt; |
| S16 off; |
| uint8x8_t src0_8x8b, src1_8x8b, src2_8x8b, src3_8x8b; |
| int16x8_t src0_8x16b, src1_8x16b, src2_8x16b, src3_8x16b; |
| int16x8_t src4_8x16b, src5_8x16b, src6_8x16b, src7_8x16b, off_8x16b; |
| int32x4_t dst0_4x32b, dst1_4x32b, dst2_4x32b, dst3_4x32b; |
| int32x4_t dst4_4x32b, dst5_4x32b, dst6_4x32b, dst7_4x32b; |
| int32x4_t add_4x32b, inv_wt_4x32b; |
| U08 ref; |
| int32x4_t log_wdc = vdupq_n_s32(ps_wt_inp_prms->wpred_log_wdc); |
| |
| src0_8x8b = vld1_u8((pu1_src + 0 * src_stride)); |
| src1_8x8b = vld1_u8((pu1_src + 1 * src_stride)); |
| src2_8x8b = vld1_u8((pu1_src + 2 * src_stride)); |
| src3_8x8b = vld1_u8((pu1_src + 3 * src_stride)); |
| /* Store */ |
| vst1_u8((pu1_dst + 0 * dst_stride), src0_8x8b); |
| vst1_u8((pu1_dst + 1 * dst_stride), src1_8x8b); |
| vst1_u8((pu1_dst + 2 * dst_stride), src2_8x8b); |
| vst1_u8((pu1_dst + 3 * dst_stride), src3_8x8b); |
| |
| if(u1_num_valid_refs) |
| { |
| /* Right 4x4 Block */ |
| src0_8x16b = vreinterpretq_s16_u16(vmovl_u8(src0_8x8b)); |
| src1_8x16b = vreinterpretq_s16_u16(vmovl_u8(src1_8x8b)); |
| src2_8x16b = vreinterpretq_s16_u16(vmovl_u8(src2_8x8b)); |
| src3_8x16b = vreinterpretq_s16_u16(vmovl_u8(src3_8x8b)); |
| |
| /* add value */ |
| add_4x32b = vdupq_n_s32(0x4000); |
| } |
| |
| /* Run thro all ref ids, except ref==num_ref, which is already done */ |
| for(ref = 0; ref < u1_num_valid_refs; ref++) |
| { |
| S32 i4_ref_idx = ai4_wt_refs[ref]; |
| |
| /* InvWt and off specific to this ref id */ |
| inv_wt = ps_wt_inp_prms->a_inv_wpred_wt[i4_ref_idx]; |
| off = (S16)ps_wt_inp_prms->a_wpred_off[i4_ref_idx]; |
| |
| /* set1 uses multiple instructions : Try to AVOID it */ |
| off_8x16b = vdupq_n_s16(off); |
| inv_wt_4x32b = vdupq_n_s32(inv_wt); |
| |
| /* Each ref id may have differnet wt/offset. */ |
| /* So we have unique inp buf for each ref id */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[i4_ref_idx]; |
| |
| /* inp - off */ |
| src4_8x16b = vsubq_s16(src0_8x16b, off_8x16b); |
| src5_8x16b = vsubq_s16(src1_8x16b, off_8x16b); |
| src6_8x16b = vsubq_s16(src2_8x16b, off_8x16b); |
| src7_8x16b = vsubq_s16(src3_8x16b, off_8x16b); |
| |
| dst0_4x32b = vmovl_s16(vget_low_s16(src4_8x16b)); |
| dst1_4x32b = vmovl_s16(vget_low_s16(src5_8x16b)); |
| dst2_4x32b = vmovl_s16(vget_low_s16(src6_8x16b)); |
| dst3_4x32b = vmovl_s16(vget_low_s16(src7_8x16b)); |
| |
| dst4_4x32b = vmovl_s16(vget_high_s16(src4_8x16b)); |
| dst5_4x32b = vmovl_s16(vget_high_s16(src5_8x16b)); |
| dst6_4x32b = vmovl_s16(vget_high_s16(src6_8x16b)); |
| dst7_4x32b = vmovl_s16(vget_high_s16(src7_8x16b)); |
| |
| /* (inp-off) << shift */ |
| dst0_4x32b = vshlq_s32(dst0_4x32b, log_wdc); |
| dst1_4x32b = vshlq_s32(dst1_4x32b, log_wdc); |
| dst2_4x32b = vshlq_s32(dst2_4x32b, log_wdc); |
| dst3_4x32b = vshlq_s32(dst3_4x32b, log_wdc); |
| |
| /* (inp-off) << shift */ |
| dst4_4x32b = vshlq_s32(dst4_4x32b, log_wdc); |
| dst5_4x32b = vshlq_s32(dst5_4x32b, log_wdc); |
| dst6_4x32b = vshlq_s32(dst6_4x32b, log_wdc); |
| dst7_4x32b = vshlq_s32(dst7_4x32b, log_wdc); |
| |
| /* ((inp-off) << shift) * inv_wt + 1<<14 */ |
| dst0_4x32b = vmlaq_s32(add_4x32b, dst0_4x32b, inv_wt_4x32b); |
| dst1_4x32b = vmlaq_s32(add_4x32b, dst1_4x32b, inv_wt_4x32b); |
| dst2_4x32b = vmlaq_s32(add_4x32b, dst2_4x32b, inv_wt_4x32b); |
| dst3_4x32b = vmlaq_s32(add_4x32b, dst3_4x32b, inv_wt_4x32b); |
| |
| /* ((inp-off) << shift) * inv_wt + 1<<14 */ |
| dst4_4x32b = vmlaq_s32(add_4x32b, dst4_4x32b, inv_wt_4x32b); |
| dst5_4x32b = vmlaq_s32(add_4x32b, dst5_4x32b, inv_wt_4x32b); |
| dst6_4x32b = vmlaq_s32(add_4x32b, dst6_4x32b, inv_wt_4x32b); |
| dst7_4x32b = vmlaq_s32(add_4x32b, dst7_4x32b, inv_wt_4x32b); |
| |
| /* (((inp-off) << shift) * inv_wt + 1<<14) >> 15 */ |
| src4_8x16b = vcombine_s16( |
| vshrn_n_s32(dst0_4x32b, IHEVCE_WT_PRED_SHIFT), |
| vshrn_n_s32(dst4_4x32b, IHEVCE_WT_PRED_SHIFT)); |
| src5_8x16b = vcombine_s16( |
| vshrn_n_s32(dst1_4x32b, IHEVCE_WT_PRED_SHIFT), |
| vshrn_n_s32(dst5_4x32b, IHEVCE_WT_PRED_SHIFT)); |
| src6_8x16b = vcombine_s16( |
| vshrn_n_s32(dst2_4x32b, IHEVCE_WT_PRED_SHIFT), |
| vshrn_n_s32(dst6_4x32b, IHEVCE_WT_PRED_SHIFT)); |
| src7_8x16b = vcombine_s16( |
| vshrn_n_s32(dst3_4x32b, IHEVCE_WT_PRED_SHIFT), |
| vshrn_n_s32(dst7_4x32b, IHEVCE_WT_PRED_SHIFT)); |
| /* Store */ |
| vst1_u8((pu1_dst + 0 * dst_stride), vqmovun_s16(src4_8x16b)); |
| vst1_u8((pu1_dst + 1 * dst_stride), vqmovun_s16(src5_8x16b)); |
| vst1_u8((pu1_dst + 2 * dst_stride), vqmovun_s16(src6_8x16b)); |
| vst1_u8((pu1_dst + 3 * dst_stride), vqmovun_s16(src7_8x16b)); |
| } |
| } |
| |
| void hme_get_wt_inp_8x8_neon( |
| layer_ctxt_t *ps_curr_layer, |
| wgt_pred_ctxt_t *ps_wt_inp_prms, |
| S32 dst_stride, |
| S32 pos_x, |
| S32 pos_y, |
| S32 size, |
| S32 num_ref, |
| U08 u1_is_wt_pred_on) |
| { |
| WORD32 ref; |
| UWORD8 *pu1_src, *pu1_dst; |
| WORD32 x_count, y_count; |
| WORD32 src_stride = ps_curr_layer->i4_inp_stride; |
| |
| /* Make sure the start positions of block are inside frame limits */ |
| pos_x = MIN(pos_x, ps_curr_layer->i4_wd - 1); |
| pos_y = MIN(pos_y, ps_curr_layer->i4_ht - 1); |
| |
| /* In case we handle imcomplete CTBs, we copy only as much as reqd */ |
| /* from input buffers to prevent out of bound accesses. In this */ |
| /* case, we do padding in x or y or both dirns */ |
| x_count = MIN(size, (ps_curr_layer->i4_wd - pos_x)); |
| y_count = MIN(size, (ps_curr_layer->i4_ht - pos_y)); |
| |
| /* Fixed source */ |
| pu1_src = ps_curr_layer->pu1_inp; |
| pu1_src += (pos_x + (pos_y * src_stride)); |
| |
| if(!u1_is_wt_pred_on) |
| { |
| uint8x8_t src0_8x8b, src1_8x8b, src2_8x8b, src3_8x8b; |
| |
| /************* Top 4x8 Processing ****************/ |
| /* Load Source : Lower 64 bit */ |
| src0_8x8b = vld1_u8(pu1_src + 0 * src_stride); |
| src1_8x8b = vld1_u8(pu1_src + 1 * src_stride); |
| src2_8x8b = vld1_u8(pu1_src + 2 * src_stride); |
| src3_8x8b = vld1_u8(pu1_src + 3 * src_stride); |
| |
| /* ref==num_ref */ /* last ref will be non weighted input */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| /* Store */ |
| vst1_u8((pu1_dst + 0 * dst_stride), src0_8x8b); |
| vst1_u8((pu1_dst + 1 * dst_stride), src1_8x8b); |
| vst1_u8((pu1_dst + 2 * dst_stride), src2_8x8b); |
| vst1_u8((pu1_dst + 3 * dst_stride), src3_8x8b); |
| |
| /************* Bottom 4x8 Processing ****************/ |
| pu1_src += 4 * src_stride; |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref] + 4 * dst_stride; |
| |
| /* Load Source : Lower 64 bit */ |
| src0_8x8b = vld1_u8(pu1_src + 0 * src_stride); |
| src1_8x8b = vld1_u8(pu1_src + 1 * src_stride); |
| src2_8x8b = vld1_u8(pu1_src + 2 * src_stride); |
| src3_8x8b = vld1_u8(pu1_src + 3 * src_stride); |
| /* ref==num_ref */ /* last ref will be non weighted input */ |
| /* Store */ |
| vst1_u8((pu1_dst + 0 * dst_stride), src0_8x8b); |
| vst1_u8((pu1_dst + 1 * dst_stride), src1_8x8b); |
| vst1_u8((pu1_dst + 2 * dst_stride), src2_8x8b); |
| vst1_u8((pu1_dst + 3 * dst_stride), src3_8x8b); |
| |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| |
| if(x_count != size) |
| { |
| hme_pad_right(pu1_dst + x_count - 1, dst_stride, size - x_count, y_count); |
| } |
| |
| /* Check and do padding in bottom directino if need be */ |
| if(y_count != size) |
| { |
| hme_pad_bot(pu1_dst + (y_count - 1) * dst_stride, dst_stride, size - y_count, size); |
| } |
| |
| for(ref = 0; ref < num_ref + 1; ref++) |
| { |
| ps_wt_inp_prms->apu1_wt_inp[ref] = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| } |
| } |
| else |
| { |
| S32 wt, off; |
| S32 ai4_wt_refs[MAX_NUM_REF]; |
| U08 u1_num_valid_refs = 0; |
| |
| for(ref = 0; ref < num_ref; ref++) |
| { |
| wt = ps_wt_inp_prms->a_wpred_wt[ref]; |
| off = ps_wt_inp_prms->a_wpred_off[ref]; |
| |
| if((WGHT_DEFAULT == wt) && (0 == off)) |
| { |
| ps_wt_inp_prms->apu1_wt_inp[ref] = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| } |
| else |
| { |
| ai4_wt_refs[u1_num_valid_refs++] = ref; |
| ps_wt_inp_prms->apu1_wt_inp[ref] = ps_wt_inp_prms->apu1_wt_inp_buf_array[ref]; |
| } |
| } |
| |
| ps_wt_inp_prms->apu1_wt_inp[num_ref] = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| |
| /************* Top 4x8 Processing ****************/ |
| /* ref==num_ref */ /* last ref will be non weighted input */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[num_ref]; |
| ihevce_get_wt_inp_4x8_neon( |
| pu1_src, |
| pu1_dst, |
| ps_wt_inp_prms, |
| u1_num_valid_refs, |
| ai4_wt_refs, |
| src_stride, |
| dst_stride); |
| /************* Bottom 4x8 Processing ****************/ |
| pu1_src += 4 * src_stride; |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[num_ref] + 4 * dst_stride; |
| ihevce_get_wt_inp_4x8_neon( |
| pu1_src, |
| pu1_dst, |
| ps_wt_inp_prms, |
| u1_num_valid_refs, |
| ai4_wt_refs, |
| src_stride, |
| dst_stride); |
| |
| for(ref = 0; ref < u1_num_valid_refs; ref++) |
| { |
| /* Check and do padding in right direction if need be */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[ai4_wt_refs[ref]]; |
| if(x_count != size) |
| { |
| hme_pad_right(pu1_dst + x_count - 1, dst_stride, size - x_count, y_count); |
| } |
| |
| /* Check and do padding in bottom directino if need be */ |
| if(y_count != size) |
| { |
| hme_pad_bot(pu1_dst + (y_count - 1) * dst_stride, dst_stride, size - y_count, size); |
| } |
| } |
| |
| /* Check and do padding in right direction if need be */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[num_ref]; |
| if(x_count != size) |
| { |
| hme_pad_right(pu1_dst + x_count - 1, dst_stride, size - x_count, y_count); |
| } |
| |
| /* Check and do padding in bottom directino if need be */ |
| if(y_count != size) |
| { |
| hme_pad_bot(pu1_dst + (y_count - 1) * dst_stride, dst_stride, size - y_count, size); |
| } |
| } |
| } |
| |
| void hme_get_wt_inp_ctb_neon( |
| layer_ctxt_t *ps_curr_layer, |
| wgt_pred_ctxt_t *ps_wt_inp_prms, |
| S32 dst_stride, |
| S32 pos_x, |
| S32 pos_y, |
| S32 size, |
| S32 num_ref, |
| U08 u1_is_wt_pred_on) |
| { |
| WORD32 ref, i, j; |
| UWORD8 *pu1_src, *pu1_dst; |
| WORD32 x_count, y_count; |
| WORD32 src_stride = ps_curr_layer->i4_inp_stride; |
| |
| /* In case we handle imcomplete CTBs, we copy only as much as reqd */ |
| /* from input buffers to prevent out of bound accesses. In this */ |
| /* case, we do padding in x or y or both dirns */ |
| x_count = MIN(size, (ps_curr_layer->i4_wd - pos_x)); |
| y_count = MIN(size, (ps_curr_layer->i4_ht - pos_y)); |
| |
| /* Fixed source */ |
| pu1_src = ps_curr_layer->pu1_inp; |
| pu1_src += (pos_x + (pos_y * src_stride)); |
| |
| if(!u1_is_wt_pred_on) |
| { |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| |
| if(0 == (x_count & 15)) |
| { |
| uint8x16_t src0_16x8b, src1_16x8b, src2_16x8b, src3_16x8b; |
| |
| for(i = 0; i < y_count; i += 4) /* 4 rows at a time */ |
| { |
| for(j = 0; j < x_count; j += 16) /* 16 cols at a time */ |
| { |
| /* Load 4x16 Source */ |
| src0_16x8b = vld1q_u8(pu1_src + 0 * src_stride); |
| src1_16x8b = vld1q_u8(pu1_src + 1 * src_stride); |
| src2_16x8b = vld1q_u8(pu1_src + 2 * src_stride); |
| src3_16x8b = vld1q_u8(pu1_src + 3 * src_stride); |
| |
| /* ref==num_ref */ /* last ref will be non weighted input */ |
| /* Store */ |
| vst1q_u8((pu1_dst + 0 * dst_stride), src0_16x8b); |
| vst1q_u8((pu1_dst + 1 * dst_stride), src1_16x8b); |
| vst1q_u8((pu1_dst + 2 * dst_stride), src2_16x8b); |
| vst1q_u8((pu1_dst + 3 * dst_stride), src3_16x8b); |
| |
| pu1_src += 16; |
| pu1_dst += 16; |
| } |
| |
| pu1_src = pu1_src - x_count + 4 * src_stride; |
| pu1_dst = pu1_dst - x_count + 4 * dst_stride; |
| } |
| } |
| else if(0 == (x_count & 7)) /* wd multiple of 8 case */ |
| { |
| uint8x8_t src0_8x8b, src1_8x8b, src2_8x8b, src3_8x8b; |
| for(i = 0; i < y_count; i += 4) /* 4 rows at a time */ |
| { |
| for(j = 0; j < x_count; j += 8) /* 8 cols at a time */ |
| { |
| /* Load 4x8 Source */ |
| src0_8x8b = vld1_u8(pu1_src + 0 * src_stride); |
| src1_8x8b = vld1_u8(pu1_src + 1 * src_stride); |
| src2_8x8b = vld1_u8(pu1_src + 2 * src_stride); |
| src3_8x8b = vld1_u8(pu1_src + 3 * src_stride); |
| |
| /* ref==num_ref */ /* last ref will be non weighted input */ |
| /* Store */ |
| vst1_u8((pu1_dst + 0 * dst_stride), src0_8x8b); |
| vst1_u8((pu1_dst + 1 * dst_stride), src1_8x8b); |
| vst1_u8((pu1_dst + 2 * dst_stride), src2_8x8b); |
| vst1_u8((pu1_dst + 3 * dst_stride), src3_8x8b); |
| |
| pu1_src += 8; |
| pu1_dst += 8; |
| } |
| |
| pu1_src = pu1_src - x_count + 4 * src_stride; |
| pu1_dst = pu1_dst - x_count + 4 * dst_stride; |
| } |
| } |
| else /* wd multiple of 4 case */ |
| { |
| for(i = 0; i < y_count; i += 4) /* 4 rows at a time */ |
| { |
| for(j = 0; j < x_count; j += 4) /* 4 cols at a time */ |
| { |
| /* ref==num_ref */ /* last ref will be non weighted input */ |
| *(WORD32 *)(&pu1_dst[0 * dst_stride]) = *(WORD32 *)(&pu1_src[0 * src_stride]); |
| *(WORD32 *)(&pu1_dst[1 * dst_stride]) = *(WORD32 *)(&pu1_src[1 * src_stride]); |
| *(WORD32 *)(&pu1_dst[2 * dst_stride]) = *(WORD32 *)(&pu1_src[2 * src_stride]); |
| *(WORD32 *)(&pu1_dst[3 * dst_stride]) = *(WORD32 *)(&pu1_src[3 * src_stride]); |
| |
| pu1_src += 4; |
| pu1_dst += 4; |
| } |
| |
| pu1_src -= x_count + 4 * src_stride; |
| pu1_dst = pu1_dst - x_count + 4 * dst_stride; |
| } |
| } |
| |
| for(i = 0; i < num_ref + 1; i++) |
| { |
| ps_wt_inp_prms->apu1_wt_inp[i] = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| } |
| |
| /* Padding */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[num_ref]; |
| |
| if(x_count != size) |
| { |
| hme_pad_right(pu1_dst + x_count - 1, dst_stride, size - x_count, y_count); |
| } |
| |
| /* Check and do padding in bottom directino if need be */ |
| if(y_count != size) |
| { |
| hme_pad_bot(pu1_dst + (y_count - 1) * dst_stride, dst_stride, size - y_count, size); |
| } |
| } |
| else |
| { |
| S32 ai4_wt_refs[MAX_NUM_REF]; |
| U08 u1_num_valid_refs = 0; |
| int32x4_t dst0_4x32b, dst1_4x32b, dst2_4x32b, dst3_4x32b; |
| int32x4_t inv_wt_4x32b, off_4x32b; |
| int16x8_t src0_8x16b, src1_8x16b, src2_8x16b, src3_8x16b; |
| |
| /* add value */ |
| int32x4_t add_4x32b = vdupq_n_s32(0x4000); |
| int32x4_t log_wdc = vdupq_n_s32(ps_wt_inp_prms->wpred_log_wdc); |
| |
| for(i = 0; i < num_ref; i++) |
| { |
| if((WGHT_DEFAULT == (ps_wt_inp_prms->a_wpred_wt[i])) && |
| (0 == (ps_wt_inp_prms->a_wpred_off[i]))) |
| { |
| ps_wt_inp_prms->apu1_wt_inp[i] = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| } |
| else |
| { |
| ai4_wt_refs[u1_num_valid_refs++] = i; |
| ps_wt_inp_prms->apu1_wt_inp[i] = ps_wt_inp_prms->apu1_wt_inp_buf_array[i]; |
| } |
| } |
| |
| ps_wt_inp_prms->apu1_wt_inp[num_ref] = ps_wt_inp_prms->apu1_wt_inp_buf_array[num_ref]; |
| |
| if(0 == (x_count & 7)) /* wd multiple of 8 case */ |
| { |
| uint8x8_t src0_8x8b, src1_8x8b, src2_8x8b, src3_8x8b; |
| int16x8_t src4_8x16b, src5_8x16b, src6_8x16b, src7_8x16b, off_8x16b; |
| int32x4_t dst4_4x32b, dst5_4x32b, dst6_4x32b, dst7_4x32b; |
| |
| for(i = 0; i < y_count; i += 4) /* 4 rows at a time */ |
| { |
| for(j = 0; j < x_count; j += 8) /* 8 cols at a time */ |
| { |
| /* Load 4x8 Source */ |
| /* Load Source : Lower 32 bit, Upper 32 bit neglected */ |
| src0_8x8b = vld1_u8(pu1_src + 0 * src_stride); |
| src1_8x8b = vld1_u8(pu1_src + 1 * src_stride); |
| src2_8x8b = vld1_u8(pu1_src + 2 * src_stride); |
| src3_8x8b = vld1_u8(pu1_src + 3 * src_stride); |
| |
| /* ref==num_ref */ /* last ref will be non weighted input */ |
| pu1_dst = (ps_wt_inp_prms->apu1_wt_inp[num_ref]) + (i * dst_stride) + j; |
| |
| /* Store */ |
| vst1_u8((pu1_dst + 0 * dst_stride), src0_8x8b); |
| vst1_u8((pu1_dst + 1 * dst_stride), src1_8x8b); |
| vst1_u8((pu1_dst + 2 * dst_stride), src2_8x8b); |
| vst1_u8((pu1_dst + 3 * dst_stride), src3_8x8b); |
| |
| if(u1_num_valid_refs) |
| { |
| /* Right 4x4 Block */ |
| src0_8x16b = vreinterpretq_s16_u16(vmovl_u8(src0_8x8b)); |
| src1_8x16b = vreinterpretq_s16_u16(vmovl_u8(src1_8x8b)); |
| src2_8x16b = vreinterpretq_s16_u16(vmovl_u8(src2_8x8b)); |
| src3_8x16b = vreinterpretq_s16_u16(vmovl_u8(src3_8x8b)); |
| } |
| |
| /* Run thro all ref ids, except ref==num_ref, which is already done */ |
| for(ref = 0; ref < u1_num_valid_refs; ref++) |
| { |
| U08 u1_ref_idx = ai4_wt_refs[ref]; |
| |
| /* Each ref id may have differnet wt/offset. */ |
| /* So we have unique inp buf for each ref id */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[u1_ref_idx] + (i * dst_stride) + j; |
| |
| /* InvWt and off specific to this ref id */ |
| off_8x16b = vdupq_n_s16(ps_wt_inp_prms->a_wpred_off[u1_ref_idx]); |
| inv_wt_4x32b = vdupq_n_s32(ps_wt_inp_prms->a_inv_wpred_wt[u1_ref_idx]); |
| |
| /* inp - off */ |
| src4_8x16b = vsubq_s16(src0_8x16b, off_8x16b); |
| src5_8x16b = vsubq_s16(src1_8x16b, off_8x16b); |
| src6_8x16b = vsubq_s16(src2_8x16b, off_8x16b); |
| src7_8x16b = vsubq_s16(src3_8x16b, off_8x16b); |
| |
| dst0_4x32b = vmovl_s16(vget_low_s16(src4_8x16b)); |
| dst1_4x32b = vmovl_s16(vget_low_s16(src5_8x16b)); |
| dst2_4x32b = vmovl_s16(vget_low_s16(src6_8x16b)); |
| dst3_4x32b = vmovl_s16(vget_low_s16(src7_8x16b)); |
| |
| dst4_4x32b = vmovl_s16(vget_high_s16(src4_8x16b)); |
| dst5_4x32b = vmovl_s16(vget_high_s16(src5_8x16b)); |
| dst6_4x32b = vmovl_s16(vget_high_s16(src6_8x16b)); |
| dst7_4x32b = vmovl_s16(vget_high_s16(src7_8x16b)); |
| |
| /* (inp-off) << shift */ |
| dst0_4x32b = vshlq_s32(dst0_4x32b, log_wdc); |
| dst1_4x32b = vshlq_s32(dst1_4x32b, log_wdc); |
| dst2_4x32b = vshlq_s32(dst2_4x32b, log_wdc); |
| dst3_4x32b = vshlq_s32(dst3_4x32b, log_wdc); |
| |
| /* (inp-off) << shift */ |
| dst4_4x32b = vshlq_s32(dst4_4x32b, log_wdc); |
| dst5_4x32b = vshlq_s32(dst5_4x32b, log_wdc); |
| dst6_4x32b = vshlq_s32(dst6_4x32b, log_wdc); |
| dst7_4x32b = vshlq_s32(dst7_4x32b, log_wdc); |
| |
| /* ((inp-off) << shift) * inv_wt + 1<<14 */ |
| dst0_4x32b = vmlaq_s32(add_4x32b, dst0_4x32b, inv_wt_4x32b); |
| dst1_4x32b = vmlaq_s32(add_4x32b, dst1_4x32b, inv_wt_4x32b); |
| dst2_4x32b = vmlaq_s32(add_4x32b, dst2_4x32b, inv_wt_4x32b); |
| dst3_4x32b = vmlaq_s32(add_4x32b, dst3_4x32b, inv_wt_4x32b); |
| |
| /* ((inp-off) << shift) * inv_wt + 1<<14 */ |
| dst4_4x32b = vmlaq_s32(add_4x32b, dst4_4x32b, inv_wt_4x32b); |
| dst5_4x32b = vmlaq_s32(add_4x32b, dst5_4x32b, inv_wt_4x32b); |
| dst6_4x32b = vmlaq_s32(add_4x32b, dst6_4x32b, inv_wt_4x32b); |
| dst7_4x32b = vmlaq_s32(add_4x32b, dst7_4x32b, inv_wt_4x32b); |
| |
| /* (((inp-off) << shift) * inv_wt + 1<<14) >> 15 */ |
| src4_8x16b = vcombine_s16( |
| vshrn_n_s32(dst0_4x32b, IHEVCE_WT_PRED_SHIFT), |
| vshrn_n_s32(dst4_4x32b, IHEVCE_WT_PRED_SHIFT)); |
| src5_8x16b = vcombine_s16( |
| vshrn_n_s32(dst1_4x32b, IHEVCE_WT_PRED_SHIFT), |
| vshrn_n_s32(dst5_4x32b, IHEVCE_WT_PRED_SHIFT)); |
| src6_8x16b = vcombine_s16( |
| vshrn_n_s32(dst2_4x32b, IHEVCE_WT_PRED_SHIFT), |
| vshrn_n_s32(dst6_4x32b, IHEVCE_WT_PRED_SHIFT)); |
| src7_8x16b = vcombine_s16( |
| vshrn_n_s32(dst3_4x32b, IHEVCE_WT_PRED_SHIFT), |
| vshrn_n_s32(dst7_4x32b, IHEVCE_WT_PRED_SHIFT)); |
| /* Store */ |
| vst1_u8((pu1_dst + 0 * dst_stride), vqmovun_s16(src4_8x16b)); |
| vst1_u8((pu1_dst + 1 * dst_stride), vqmovun_s16(src5_8x16b)); |
| vst1_u8((pu1_dst + 2 * dst_stride), vqmovun_s16(src6_8x16b)); |
| vst1_u8((pu1_dst + 3 * dst_stride), vqmovun_s16(src7_8x16b)); |
| } |
| /* Pointer update */ |
| pu1_src += 8; |
| } |
| /* Pointer update */ |
| pu1_src = pu1_src - x_count + 4 * src_stride; |
| } |
| } |
| else /* wd multiple of 4 case */ |
| { |
| uint8x16_t src0_16x8b; |
| int32x4_t src0_4x32b, src1_4x32b, src2_4x32b, src3_4x32b; |
| WORD32 dst0, dst1, dst2, dst3; |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[num_ref]; |
| for(i = 0; i < y_count; i += 4) /* 4 rows at a time */ |
| { |
| for(j = 0; j < x_count; j += 4) /* 4 cols at a time */ |
| { |
| /* ref==num_ref */ /* last ref will be non weighted input */ |
| |
| *(WORD32 *)(&pu1_dst[0 * dst_stride]) = *(WORD32 *)(&pu1_src[0 * src_stride]); |
| *(WORD32 *)(&pu1_dst[1 * dst_stride]) = *(WORD32 *)(&pu1_src[1 * src_stride]); |
| *(WORD32 *)(&pu1_dst[2 * dst_stride]) = *(WORD32 *)(&pu1_src[2 * src_stride]); |
| *(WORD32 *)(&pu1_dst[3 * dst_stride]) = *(WORD32 *)(&pu1_src[3 * src_stride]); |
| |
| /* Pointer update */ |
| pu1_src += 4; |
| pu1_dst += 4; |
| } |
| /* Pointer update */ |
| pu1_src = pu1_src - x_count + 4 * src_stride; |
| pu1_dst = pu1_dst - x_count + 4 * dst_stride; |
| } |
| |
| if(u1_num_valid_refs) |
| { |
| pu1_src = ps_curr_layer->pu1_inp; |
| pu1_src += (pos_x + (pos_y * src_stride)); |
| |
| /* Run thro all ref ids, except ref==num_ref, which is already done */ |
| for(ref = 0; ref < u1_num_valid_refs; ref++) |
| { |
| U08 u1_ref_idx = ai4_wt_refs[ref]; |
| |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[u1_ref_idx]; |
| |
| /* InvWt and off specific to this ref id */ |
| off_4x32b = vdupq_n_s32(ps_wt_inp_prms->a_wpred_off[u1_ref_idx]); |
| inv_wt_4x32b = vdupq_n_s32(ps_wt_inp_prms->a_inv_wpred_wt[u1_ref_idx]); |
| |
| for(i = 0; i < y_count; i += 4) /* 4 rows at a time */ |
| { |
| for(j = 0; j < x_count; j += 4) /* 4 cols at a time */ |
| { |
| src0_16x8b = load_unaligned_u8q(pu1_src, src_stride); |
| |
| src0_8x16b = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(src0_16x8b))); |
| src1_8x16b = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(src0_16x8b))); |
| |
| src0_4x32b = vmovl_s16(vget_low_s16(src0_8x16b)); |
| src1_4x32b = vmovl_s16(vget_high_s16(src0_8x16b)); |
| src2_4x32b = vmovl_s16(vget_low_s16(src1_8x16b)); |
| src3_4x32b = vmovl_s16(vget_high_s16(src1_8x16b)); |
| |
| /* inp - off */ |
| dst0_4x32b = vsubq_s32(src0_4x32b, off_4x32b); |
| dst1_4x32b = vsubq_s32(src1_4x32b, off_4x32b); |
| dst2_4x32b = vsubq_s32(src2_4x32b, off_4x32b); |
| dst3_4x32b = vsubq_s32(src3_4x32b, off_4x32b); |
| |
| /* (inp-off) << shift */ |
| dst0_4x32b = vshlq_s32(dst0_4x32b, log_wdc); |
| dst1_4x32b = vshlq_s32(dst1_4x32b, log_wdc); |
| dst2_4x32b = vshlq_s32(dst2_4x32b, log_wdc); |
| dst3_4x32b = vshlq_s32(dst3_4x32b, log_wdc); |
| |
| /* ((inp-off) << shift) * inv_wt */ |
| dst0_4x32b = vmlaq_s32(add_4x32b, dst0_4x32b, inv_wt_4x32b); |
| dst1_4x32b = vmlaq_s32(add_4x32b, dst1_4x32b, inv_wt_4x32b); |
| dst2_4x32b = vmlaq_s32(add_4x32b, dst2_4x32b, inv_wt_4x32b); |
| dst3_4x32b = vmlaq_s32(add_4x32b, dst3_4x32b, inv_wt_4x32b); |
| |
| /* (((inp-off) << shift) * inv_wt + 1<<14) >> 15 */ |
| dst0 = (WORD32)vget_lane_u64( |
| vreinterpret_u64_u16( |
| vqshrun_n_s32(dst0_4x32b, IHEVCE_WT_PRED_SHIFT)), |
| 0); |
| dst1 = (WORD32)vget_lane_u64( |
| vreinterpret_u64_u16( |
| vqshrun_n_s32(dst1_4x32b, IHEVCE_WT_PRED_SHIFT)), |
| 0); |
| dst2 = (WORD32)vget_lane_u64( |
| vreinterpret_u64_u16( |
| vqshrun_n_s32(dst2_4x32b, IHEVCE_WT_PRED_SHIFT)), |
| 0); |
| dst3 = (WORD32)vget_lane_u64( |
| vreinterpret_u64_u16( |
| vqshrun_n_s32(dst3_4x32b, IHEVCE_WT_PRED_SHIFT)), |
| 0); |
| |
| *(WORD32 *)(&pu1_dst[0 * dst_stride]) = dst0; |
| *(WORD32 *)(&pu1_dst[1 * dst_stride]) = dst1; |
| *(WORD32 *)(&pu1_dst[2 * dst_stride]) = dst2; |
| *(WORD32 *)(&pu1_dst[3 * dst_stride]) = dst3; |
| |
| /* Pointer update */ |
| pu1_src += 4; |
| pu1_dst += 4; |
| } |
| /* Pointer update */ |
| pu1_src = pu1_src - x_count + 4 * src_stride; |
| pu1_dst = pu1_dst - x_count + 4 * dst_stride; |
| } |
| } |
| } |
| } |
| |
| /* Padding */ |
| for(ref = 0; ref < u1_num_valid_refs; ref++) |
| { |
| /* Check and do padding in right direction if need be */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[ai4_wt_refs[ref]]; |
| if(x_count != size) |
| { |
| hme_pad_right(pu1_dst + x_count - 1, dst_stride, size - x_count, y_count); |
| } |
| |
| /* Check and do padding in bottom directino if need be */ |
| if(y_count != size) |
| { |
| hme_pad_bot(pu1_dst + (y_count - 1) * dst_stride, dst_stride, size - y_count, size); |
| } |
| } |
| |
| /* Check and do padding in right direction if need be */ |
| pu1_dst = ps_wt_inp_prms->apu1_wt_inp[num_ref]; |
| |
| if(x_count != size) |
| { |
| hme_pad_right(pu1_dst + x_count - 1, dst_stride, size - x_count, y_count); |
| } |
| |
| /* Check and do padding in bottom directino if need be */ |
| if(y_count != size) |
| { |
| hme_pad_bot(pu1_dst + (y_count - 1) * dst_stride, dst_stride, size - y_count, size); |
| } |
| } |
| } |
