blob: af6e66c4f900d76b460d0b11c61b43f872d87b1d [file] [log] [blame]
/******************************************************************************
*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*****************************************************************************
* Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore
*/
/**
*******************************************************************************
* @file
* ihevc_resi_trans_neon.c
*
* @brief
* Contains definitions of functions for computing residue and fwd transform
*
* @author
* Ittiam
*
* @par List of Functions:
* - ihevc_resi_trans_4x4_neon()
* - ihevc_resi_trans_4x4_ttype1_neon()
* - ihevc_resi_trans_8x8_neon()
* - ihevc_resi_trans_16x16_neon()
* @remarks
* None
*
*******************************************************************************
*/
/*****************************************************************************/
/* File Includes */
/*****************************************************************************/
/* System include files */
#include <stdio.h>
#include <string.h>
/* System user files */
#include "ihevc_typedefs.h"
#include "ihevc_macros.h"
#include "ihevc_defs.h"
#include "ihevc_cmn_utils_neon.h"
#include "ihevc_trans_tables.h"
#include "ihevc_resi_trans.h"
/*****************************************************************************/
/* Function Definitions */
/*****************************************************************************/
UWORD32 ihevc_resi_trans_4x4_neon(
UWORD8 *pu1_src,
UWORD8 *pu1_pred,
WORD32 *pi4_temp,
WORD16 *pi2_dst,
WORD32 src_strd,
WORD32 pred_strd,
WORD32 dst_strd,
CHROMA_PLANE_ID_T e_chroma_plane)
{
UWORD32 sad;
uint8x16_t inp_buf, pred_buf;
int16x8_t diff_1, diff_2;
int16x4_t diff_1_low, diff_1_high, diff_2_low, diff_2_high;
int16x8_t e_01, o_32;
int16x4_t e_0, e_1, o_0, o_1;
int32x4_t e_0_a_e_1, e_0_s_e_1;
int32x4_t temp1, temp2, temp3, temp4;
int32x4_t o_1_m_trans_10, o_1_m_trans_11;
int32x4_t e_03, e_12, o_03, o_12;
int16x4_t out_0, out_1, out_2, out_3;
uint16x8_t abs;
uint32x4_t b;
uint64x2_t c;
(void)pi4_temp;
if(e_chroma_plane == NULL_PLANE)
{
inp_buf = load_unaligned_u8q(pu1_src, src_strd);
pred_buf = load_unaligned_u8q(pu1_pred, pred_strd);
}
else
{
inp_buf = load_unaligned_u8qi(pu1_src + e_chroma_plane, src_strd);
pred_buf = load_unaligned_u8qi(pu1_pred + e_chroma_plane, pred_strd);
}
abs = vabdl_u8(vget_low_u8(inp_buf), vget_low_u8(pred_buf));
abs = vabal_u8(abs, vget_high_u8(inp_buf), vget_high_u8(pred_buf));
b = vpaddlq_u16(abs);
c = vpaddlq_u32(b);
sad = vget_lane_u32(vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
vreinterpret_u32_u64(vget_high_u64(c))),
0);
diff_1 = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(inp_buf), vget_low_u8(pred_buf)));
diff_2 = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(inp_buf), vget_high_u8(pred_buf)));
diff_1_low = vget_low_s16(diff_1);
diff_1_high = vget_high_s16(diff_1);
diff_2_low = vget_low_s16(diff_2);
diff_2_high = vget_high_s16(diff_2);
transpose_s16_4x4d(&diff_1_low, &diff_1_high, &diff_2_low, &diff_2_high);
diff_1 = vcombine_s16(diff_1_low, diff_1_high);
diff_2 = vcombine_s16(diff_2_high, diff_2_low);
e_01 = vaddq_s16(diff_1, diff_2);
o_32 = vsubq_s16(diff_1, diff_2);
e_0 = vget_low_s16(e_01);
e_1 = vget_high_s16(e_01);
o_0 = vget_high_s16(o_32);
o_1 = vget_low_s16(o_32);
e_0_a_e_1 = vaddl_s16(e_0, e_1);
e_0_s_e_1 = vsubl_s16(e_0, e_1);
temp1 = vmulq_n_s32(e_0_a_e_1, (WORD32)g_ai2_ihevc_trans_4[0][0]);
temp2 = vmulq_n_s32(e_0_s_e_1, (WORD32)g_ai2_ihevc_trans_4[0][0]);
o_1_m_trans_10 = vmull_n_s16(o_1, (WORD32)g_ai2_ihevc_trans_4[1][0]);
o_1_m_trans_11 = vmull_n_s16(o_1, (WORD32)g_ai2_ihevc_trans_4[1][1]);
temp3 = vmlal_n_s16(o_1_m_trans_10, o_0, (WORD32)g_ai2_ihevc_trans_4[1][1]);
temp4 = vmlsl_n_s16(o_1_m_trans_11, o_0, (WORD32)g_ai2_ihevc_trans_4[1][0]);
transpose_s32_4x4(&temp1, &temp3, &temp2, &temp4);
e_03 = vaddq_s32(temp1, temp4);
e_12 = vaddq_s32(temp3, temp2);
o_03 = vsubq_s32(temp1, temp4);
o_12 = vsubq_s32(temp3, temp2);
e_0_a_e_1 = vaddq_s32(e_03, e_12);
e_0_s_e_1 = vsubq_s32(e_03, e_12);
temp1 = vmulq_n_s32(e_0_a_e_1, (WORD32)g_ai2_ihevc_trans_4[0][0]);
temp2 = vmulq_n_s32(e_0_s_e_1, (WORD32)g_ai2_ihevc_trans_4[0][0]);
o_1_m_trans_10 = vmulq_n_s32(o_03, (WORD32)g_ai2_ihevc_trans_4[1][0]);
o_1_m_trans_11 = vmulq_n_s32(o_03, (WORD32)g_ai2_ihevc_trans_4[1][1]);
temp3 = vmlaq_n_s32(o_1_m_trans_10, o_12, (WORD32)g_ai2_ihevc_trans_4[1][1]);
temp4 = vmlsq_n_s32(o_1_m_trans_11, o_12, (WORD32)g_ai2_ihevc_trans_4[1][0]);
out_0 = vrshrn_n_s32(temp1, 9);
out_1 = vrshrn_n_s32(temp3, 9);
out_2 = vrshrn_n_s32(temp2, 9);
out_3 = vrshrn_n_s32(temp4, 9);
vst1_s16(pi2_dst, out_0);
vst1_s16(pi2_dst + dst_strd, out_1);
vst1_s16(pi2_dst + 2 * dst_strd, out_2);
vst1_s16(pi2_dst + 3 * dst_strd, out_3);
return sad;
}
/**
*******************************************************************************
*
* @brief
* This function performs residue calculation and forward transform type 1
* on input pixels
*
* @par Description:
* Performs residue calculation by subtracting source and prediction and
* followed by forward transform
*
* @param[in] pu1_src
* Input 4x4 pixels
*
* @param[in] pu1_pred
* Prediction data
*
* @param[in] pi2_tmp
* Temporary buffer of size 4x4
*
* @param[out] pi2_dst
* Output 4x4 coefficients
*
* @param[in] src_strd
* Input stride
*
* @param[in] pred_strd
* Prediction Stride
*
* @param[in] dst_strd
* Output Stride
*
* @param[in] e_chroma_plane
* Enum singalling chroma plane
*
* @returns block sad
*
* @remarks
* None
*
*******************************************************************************
*/
UWORD32 ihevc_resi_trans_4x4_ttype1_neon(
UWORD8 *pu1_src,
UWORD8 *pu1_pred,
WORD32 *pi4_temp,
WORD16 *pi2_dst,
WORD32 src_strd,
WORD32 pred_strd,
WORD32 dst_strd,
CHROMA_PLANE_ID_T e_chroma_plane)
{
UWORD32 sad;
int16x4_t src0_4x16b;
int16x4_t src1_4x16b;
int16x4_t src2_4x16b;
int16x4_t src3_4x16b;
int32x4_t src0_4x32b;
int32x4_t src1_4x32b;
int32x4_t src2_4x32b;
int32x4_t src3_4x32b;
/*load source and pred values */
const uint8x16_t src_u8 = load_unaligned_u8q(pu1_src, src_strd);
const uint8x16_t pred_u8 = load_unaligned_u8q(pu1_pred, pred_strd);
const int16x8_t src_reg0 =
vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(src_u8), vget_low_u8(pred_u8)));
const int16x8_t src_reg1 =
vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(src_u8), vget_high_u8(pred_u8)));
int32x4_t add_val = vdupq_n_s32(1);
uint16x8_t abs = vabdl_u8(vget_low_u8(src_u8), vget_low_u8(pred_u8));
uint32x4_t b;
uint64x2_t c;
UNUSED(e_chroma_plane);
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(pred_u8));
b = vpaddlq_u16(abs);
c = vpaddlq_u32(b);
sad = vget_lane_u32(vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
vreinterpret_u32_u64(vget_high_u64(c))),
0);
(void)pi4_temp;
/************************* 4x4 16bit Transpose ***********************/
src0_4x16b = vget_low_s16(src_reg0);
src1_4x16b = vget_high_s16(src_reg0);
src2_4x16b = vget_low_s16(src_reg1);
src3_4x16b = vget_high_s16(src_reg1);
transpose_s16_4x4d(&src0_4x16b, &src1_4x16b, &src2_4x16b, &src3_4x16b);
/************************** 4x4 Transpose End *************************/
/* Residue + Forward Transform 1st stage */
/* coeff2_4x32b = 74 74 74 74 */
const int32x4_t coeff2_4x32b =
vdupq_n_s32(74); //vld1q_s32(&g_ai4_ihevc_trans_dst_intr_4[2][0]);
/* coeff0_4x32b = 29 29 29 29 */
const int32x4_t coeff0_4x32b =
vdupq_n_s32(29); //vld1q_s32(&g_ai4_ihevc_trans_dst_intr_4[0][0]);
/* coeff1_4x32b = 55 55 55 55 */
const int32x4_t coeff1_4x32b =
vdupq_n_s32(55); //vld1q_s32(&g_ai4_ihevc_trans_dst_intr_4[1][0]);
/* c0 to c3 calculation */
int32x4_t c0_4x32b = vaddl_s16(src0_4x16b, src3_4x16b); /* r0+r3 */
int32x4_t c1_4x32b = vaddl_s16(src1_4x16b, src3_4x16b); /* r1+r3 */
int32x4_t c2_4x32b = vsubl_s16(src0_4x16b, src1_4x16b); /* r0-r1 */
int32x4_t c3_4x32b = vmulq_s32(vmovl_s16(src2_4x16b), coeff2_4x32b); /* 74*r2 */
src0_4x16b = vadd_s16(src0_4x16b, src1_4x16b); /* r0+r1 */
src1_4x32b = vsubl_s16(src0_4x16b, src3_4x16b); /* r0+r1-r3 */
src0_4x32b = vmlaq_s32(c3_4x32b, c0_4x32b, coeff0_4x32b); /* 29*c0 + c3 */
src2_4x32b = vmulq_s32(c2_4x32b, coeff0_4x32b); /* 29*c2 - c3 */
src3_4x32b = vmlaq_s32(c3_4x32b, c2_4x32b, coeff1_4x32b); /* 55*c2 + c3 */
src2_4x32b = vsubq_s32(src2_4x32b, c3_4x32b);
src0_4x32b = vmlaq_s32(src0_4x32b, c1_4x32b, coeff1_4x32b); /* 29*c0 + 55*c1 + c3 */
src2_4x32b = vmlaq_s32(src2_4x32b, c0_4x32b, coeff1_4x32b); /* 29*c2 + 55*c0 - c3 */
c1_4x32b = vmulq_s32(c1_4x32b, coeff0_4x32b); /* 55*c2 - 29*c1 + c3 */
src1_4x32b = vmulq_s32(src1_4x32b, coeff2_4x32b); /*74*(r0+r1-r3)*/
src3_4x32b = vsubq_s32(src3_4x32b, c1_4x32b);
/* result + add */
src1_4x32b = vaddq_s32(src1_4x32b, add_val);
src0_4x32b = vaddq_s32(src0_4x32b, add_val);
src2_4x32b = vaddq_s32(src2_4x32b, add_val);
src3_4x32b = vaddq_s32(src3_4x32b, add_val);
/* result >> shift */
src1_4x32b = vshrq_n_s32(src1_4x32b, 1);
src0_4x32b = vshrq_n_s32(src0_4x32b, 1);
src2_4x32b = vshrq_n_s32(src2_4x32b, 1);
src3_4x32b = vshrq_n_s32(src3_4x32b, 1);
/* Forward transform 2nd stage */
{
/************************* 4x4 32bit Transpose ***********************/
transpose_s32_4x4(&src0_4x32b, &src1_4x32b, &src2_4x32b, &src3_4x32b);
/************************** 4x4 Transpose End *************************/
/* add value */
add_val = vdupq_n_s32(128);
c0_4x32b = vaddq_s32(src0_4x32b, src3_4x32b); /* r0+r3 */
c1_4x32b = vaddq_s32(src1_4x32b, src3_4x32b); /* r1+r3 */
c2_4x32b = vsubq_s32(src0_4x32b, src1_4x32b); /* r0-r1 */
c3_4x32b = vmulq_s32(src2_4x32b, coeff2_4x32b); /* 74*r2 */
src1_4x32b = vaddq_s32(src0_4x32b, src1_4x32b); /* r0+r1 */
src1_4x32b = vsubq_s32(src1_4x32b, src3_4x32b); /* r0+r1-r3 */
src0_4x32b = vmlaq_s32(c3_4x32b, c0_4x32b, coeff0_4x32b); /* 29*c0 + c3 */
src2_4x32b = vmulq_s32(c2_4x32b, coeff0_4x32b); /* 29*c2 - c3 */
src3_4x32b = vmlaq_s32(c3_4x32b, c2_4x32b, coeff1_4x32b); /* 55*c2 + c3 */
src2_4x32b = vsubq_s32(src2_4x32b, c3_4x32b);
src0_4x32b = vmlaq_s32(src0_4x32b, c1_4x32b, coeff1_4x32b); /* 29*c0 + 55*c1 + c3 */
src2_4x32b = vmlaq_s32(src2_4x32b, c0_4x32b, coeff1_4x32b); /* 29*c2 + 55*c0 - c3 */
c1_4x32b = vmulq_s32(c1_4x32b, coeff0_4x32b); /* 55*c2 - 29*c1 + c3 */
src1_4x32b = vmulq_s32(src1_4x32b, coeff2_4x32b); /*74*(r0+r1-r3)*/
src3_4x32b = vsubq_s32(src3_4x32b, c1_4x32b);
/* result + add */
src1_4x32b = vaddq_s32(src1_4x32b, add_val);
src0_4x32b = vaddq_s32(src0_4x32b, add_val);
src2_4x32b = vaddq_s32(src2_4x32b, add_val);
src3_4x32b = vaddq_s32(src3_4x32b, add_val);
src1_4x32b = vshrq_n_s32(src1_4x32b, 8);
src0_4x32b = vshrq_n_s32(src0_4x32b, 8);
src2_4x32b = vshrq_n_s32(src2_4x32b, 8);
src3_4x32b = vshrq_n_s32(src3_4x32b, 8);
vst1_s16((pi2_dst + dst_strd), vmovn_s32(src1_4x32b));
vst1_s16(pi2_dst, vmovn_s32(src0_4x32b));
vst1_s16((pi2_dst + 2 * dst_strd), vmovn_s32(src2_4x32b));
vst1_s16((pi2_dst + 3 * dst_strd), vmovn_s32(src3_4x32b));
}
return sad;
}
/**
*******************************************************************************
*
* @brief
* This function performs residue calculation and forward transform on
* input pixels
*
* @par Description:
* Performs residue calculation by subtracting source and prediction and
* followed by forward transform
*
* @param[in] pu1_src
* Input 8x8 pixels
*
* @param[in] pu1_pred
* Prediction data
*
* @param[in] pi2_tmp
* Temporary buffer of size 8x8
*
* @param[out] pi2_dst
* Output 8x8 coefficients
*
* @param[in] src_strd
* Input stride
*
* @param[in] pred_strd
* Prediction Stride
*
* @param[in] dst_strd
* Output Stride
*
* @param[in] e_chroma_plane
* Enum singalling chroma plane
*
* @returns Void
*
* @remarks
* None
*
*******************************************************************************
*/
UWORD32 ihevc_resi_trans_8x8_neon(
UWORD8 *pu1_src,
UWORD8 *pu1_pred,
WORD32 *pi4_temp,
WORD16 *pi2_dst,
WORD32 src_strd,
WORD32 pred_strd,
WORD32 dst_strd,
CHROMA_PLANE_ID_T e_chroma_plane)
{
int16x8_t diff_16[8];
int16x8_t abs = vdupq_n_s16(0);
int32x4_t tmp_a;
int64x2_t tmp_b;
int32x2_t sad_v;
int32x4x2_t a0, a1, a2, a3, a4, a5, a6, a7;
UWORD32 sad;
(void)pi4_temp;
// stage 1
for(int k = 0; k < 8; k++)
{
if(NULL_PLANE == e_chroma_plane)
{
diff_16[k] = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(pu1_src), vld1_u8(pu1_pred)));
}
else
{
diff_16[k] = vreinterpretq_s16_u16(vsubl_u8(vld2_u8(pu1_src).val[e_chroma_plane],
vld2_u8(pu1_pred).val[e_chroma_plane]));
}
pu1_src += src_strd;
pu1_pred += pred_strd;
abs = vaddq_s16(abs, vabsq_s16(diff_16[k]));
}
tmp_a = vpaddlq_s16(abs);
tmp_b = vpaddlq_s32(tmp_a);
sad_v = vadd_s32(vreinterpret_s32_s64(vget_low_s64(tmp_b)),
vreinterpret_s32_s64(vget_high_s64(tmp_b)));
sad = vget_lane_s32(sad_v, 0);
transpose_s16_8x8(
&diff_16[0],
&diff_16[1],
&diff_16[2],
&diff_16[3],
&diff_16[4],
&diff_16[5],
&diff_16[6],
&diff_16[7]);
{
const int16x8_t o3 = vsubq_s16(diff_16[3], diff_16[4]); /*C3 - C4*/
const int16x8_t o2 = vsubq_s16(diff_16[2], diff_16[5]); /*C2 - C5*/
const int16x8_t o1 = vsubq_s16(diff_16[1], diff_16[6]); /*C1 - C6*/
const int16x8_t o0 = vsubq_s16(diff_16[0], diff_16[7]); /*C0 - C7*/
const int16x8_t e0 = vaddq_s16(diff_16[0], diff_16[7]); /*C0 + C7*/
const int16x8_t e1 = vaddq_s16(diff_16[1], diff_16[6]); /*C1 + C6*/
const int16x8_t e2 = vaddq_s16(diff_16[2], diff_16[5]); /*C2 + C5*/
const int16x8_t e3 = vaddq_s16(diff_16[3], diff_16[4]); /*C3 + C4*/
const int16x8_t ee0 = vaddq_s16(e0, e3); /*C0 + C3 + C4 + C7*/
const int16x8_t ee1 = vaddq_s16(e1, e2); /*C1 + C2 + C5 + C6*/
const int16x8_t eo0 = vsubq_s16(e0, e3); /*C0 - C3 - C4 + C7*/
const int16x8_t eo1 = vsubq_s16(e1, e2); /*C1 - C2 - C5 + C6*/
/*C0 + C1 + C2 + C3 + C4 + C5 + C6 + C7*/
const int16x8_t eee = vaddq_s16(ee1, ee0);
/*C0 - C1 - C2 + C3 + C4 - C5 - C6 + C7*/
const int16x8_t eeo = vsubq_s16(ee0, ee1);
/*F2[0] of 83*(C0 - C3 - C4 + C7)*/
a2.val[0] = vmull_n_s16(vget_low_s16(eo0), 83);
/*F6[0] of 36*(C0 - C3 - C4 + C7)*/
a6.val[0] = vmull_n_s16(vget_low_s16(eo0), 36);
/*F2[1] of 83*(C0 - C3 - C4 + C7)*/
a2.val[1] = vmull_n_s16(vget_high_s16(eo0), 83);
/*F6[1] of 36*(C0 - C3 - C4 + C7)*/
a6.val[1] = vmull_n_s16(vget_high_s16(eo0), 36);
/*F6[1] = 36*(C0 - C3 - C4 + C7) - 83*(C1 - C2 - C5 + C6)*/
a6.val[1] = vmlsl_n_s16(a6.val[1], vget_high_s16(eo1), 83);
/*F2[1] = 83*(C0 - C3 - C4 + C7) + 36*(C1 - C2 - C5 + C6)*/
a2.val[1] = vmlal_n_s16(a2.val[1], vget_high_s16(eo1), 36);
/*F6[0] = 36*(C0 - C3 - C4 + C7) - 83*(C1 - C2 - C5 + C6)*/
a6.val[0] = vmlsl_n_s16(a6.val[0], vget_low_s16(eo1), 83);
/*F2[0] = 83*(C0 - C3 - C4 + C7) + 36*(C1 - C2 - C5 + C6)*/
a2.val[0] = vmlal_n_s16(a2.val[0], vget_low_s16(eo1), 36);
/*F0[0] = 64*(C0 + C1 + C2 + C3 + C4 + C5 + C6 + C7)*/
a0.val[0] = vshll_n_s16(vget_low_s16(eee), 6);
/*F0[1] = 64*(C0 + C1 + C2 + C3 + C4 + C5 + C6 + C7)*/
a0.val[1] = vshll_n_s16(vget_high_s16(eee), 6);
/*F4[0] = 64*(C0 - C1 - C2 + C3 + C4 - C5 - C6 + C7)*/
a4.val[0] = vshll_n_s16(vget_low_s16(eeo), 6);
/*F4[1] = 64*(C0 - C1 - C2 + C3 + C4 - C5 - C6 + C7)*/
a4.val[1] = vshll_n_s16(vget_high_s16(eeo), 6);
a7.val[0] = vmull_n_s16(vget_low_s16(o0), 18); /*F7[0] = 18*(C0 - C7)*/
a5.val[0] = vmull_n_s16(vget_low_s16(o0), 50); /*F5[0] = 50*(C0 - C7)*/
a3.val[0] = vmull_n_s16(vget_low_s16(o0), 75); /*F3[0] = 75*(C0 - C7)*/
a1.val[0] = vmull_n_s16(vget_low_s16(o0), 89); /*F1[0] = 89*(C0 - C7)*/
a1.val[1] = vmull_n_s16(vget_high_s16(o0), 89); /*F1[1] = 89*(C0 - C7)*/
a3.val[1] = vmull_n_s16(vget_high_s16(o0), 75); /*F3[1] = 75*(C0 - C7)*/
a5.val[1] = vmull_n_s16(vget_high_s16(o0), 50); /*F5[1] = 50*(C0 - C7)*/
a7.val[1] = vmull_n_s16(vget_high_s16(o0), 18); /*F7[1] = 18*(C0 - C7)*/
/*F7[0] = 18*(C0 - C7) - 50*(C1 - C6)*/
a7.val[0] = vmlsl_n_s16(a7.val[0], vget_low_s16(o1), 50);
/*F5[0] = 50*(C0 - C7) - 89*(C1 - C6)*/
a5.val[0] = vmlsl_n_s16(a5.val[0], vget_low_s16(o1), 89);
/*F3[0] = 75*(C0 - C7) - 18*(C1 - C6)*/
a3.val[0] = vmlsl_n_s16(a3.val[0], vget_low_s16(o1), 18);
/*F1[0] = 89*(C0 - C7) + 75*(C1 - C6)*/
a1.val[0] = vmlal_n_s16(a1.val[0], vget_low_s16(o1), 75);
/*F1[1] = 89*(C0 - C7) + 75*(C1 - C6)*/
a1.val[1] = vmlal_n_s16(a1.val[1], vget_high_s16(o1), 75);
/*F3[1] = 75*(C0 - C7) - 18*(C1 - C6)*/
a3.val[1] = vmlsl_n_s16(a3.val[1], vget_high_s16(o1), 18);
/*F5[1] = 50*(C0 - C7) - 89*(C1 - C6)*/
a5.val[1] = vmlsl_n_s16(a5.val[1], vget_high_s16(o1), 89);
/*F7[1] = 18*(C0 - C7) - 50*(C1 - C6)*/
a7.val[1] = vmlsl_n_s16(a7.val[1], vget_high_s16(o1), 50);
/*F7[0] = 18*(C0 - C7) - 50*(C1 - C6) + 75*(C2 - C5)*/
a7.val[0] = vmlal_n_s16(a7.val[0], vget_low_s16(o2), 75);
/*F5[0] = 50*(C0 - C7) - 89*(C1 - C6) + 18*(C2 - C5)*/
a5.val[0] = vmlal_n_s16(a5.val[0], vget_low_s16(o2), 18);
/*F3[0] = 75*(C0 - C7) - 18*(C1 - C6) - 89*(C2 - C5)*/
a3.val[0] = vmlsl_n_s16(a3.val[0], vget_low_s16(o2), 89);
/*F1[0] = 89*(C0 - C7) + 75*(C1 - C6) + 50*(C2 - C5)*/
a1.val[0] = vmlal_n_s16(a1.val[0], vget_low_s16(o2), 50);
/*F1[1] = 89*(C0 - C7) + 75*(C1 - C6) + 50*(C2 - C5)*/
a1.val[1] = vmlal_n_s16(a1.val[1], vget_high_s16(o2), 50);
/*F3[1] = 75*(C0 - C7) - 18*(C1 - C6) - 89*(C2 - C5)*/
a3.val[1] = vmlsl_n_s16(a3.val[1], vget_high_s16(o2), 89);
/*F5[1] = 50*(C0 - C7) - 89*(C1 - C6) + 18*(C2 - C5)*/
a5.val[1] = vmlal_n_s16(a5.val[1], vget_high_s16(o2), 18);
/*F7[1] = 18*(C0 - C7) - 50*(C1 - C6) + 75*(C2 - C5)*/
a7.val[1] = vmlal_n_s16(a7.val[1], vget_high_s16(o2), 75);
/*F7[0] = 18*(C0 - C7) - 50*(C1 - C6) + 75*(C2 - C5) - 89*(C3 - C4)*/
a7.val[0] = vmlsl_n_s16(a7.val[0], vget_low_s16(o3), 89);
/*F5[0] = 50*(C0 - C7) - 89*(C1 - C6) + 18*(C2 - C5) + 75*(C3 - C4)*/
a5.val[0] = vmlal_n_s16(a5.val[0], vget_low_s16(o3), 75);
/*F3[0] = 75*(C0 - C7) - 18*(C1 - C6) - 89*(C2 - C5) - 50*(C3 - C4)*/
a3.val[0] = vmlsl_n_s16(a3.val[0], vget_low_s16(o3), 50);
/*F1[0] = 89*(C0 - C7) + 75*(C1 - C6) + 50*(C2 - C5) + 18*(C3 - C4)*/
a1.val[0] = vmlal_n_s16(a1.val[0], vget_low_s16(o3), 18);
/*F1[1] = 89*(C0 - C7) + 75*(C1 - C6) + 50*(C2 - C5) + 18*(C3 - C4)*/
a1.val[1] = vmlal_n_s16(a1.val[1], vget_high_s16(o3), 18);
/*F3[1] = 75*(C0 - C7) - 18*(C1 - C6) - 89*(C2 - C5) - 50*(C3 - C4)*/
a3.val[1] = vmlsl_n_s16(a3.val[1], vget_high_s16(o3), 50);
/*F5[1] = 50*(C0 - C7) - 89*(C1 - C6) + 18*(C2 - C5) + 75*(C3 - C4)*/
a5.val[1] = vmlal_n_s16(a5.val[1], vget_high_s16(o3), 75);
/*F7[1] = 18*(C0 - C7) - 50*(C1 - C6) + 75*(C2 - C5) - 89*(C3 - C4)*/
a7.val[1] = vmlsl_n_s16(a7.val[1], vget_high_s16(o3), 89);
}
//Stage 2
{
int32x4_t h0, h1, h2, h3, h4, h5, h6, h7;
int32x4_t e0_2, e1_2, e2_2, e3_2;
int32x4_t o0_2, o1_2, o2_2, o3_2;
int32x4_t ee1_2, eo1_2, eo0_2, ee0_2;
int16x4_t row0, row1, row2, row3, row4, row5, row6, row7;
/*Transposing second half of transform stage 1 (1)*/
int32x4x2_t b1 = vtrnq_s32(a0.val[1], a1.val[1]);
int32x4x2_t b3 = vtrnq_s32(a2.val[1], a3.val[1]);
int32x4x2_t b0 = vtrnq_s32(a0.val[0], a1.val[0]);
int32x4x2_t b2 = vtrnq_s32(a2.val[0], a3.val[0]);
/*Transposing second half of transform stage 1 (2)*/
a0.val[0] = vcombine_s32(vget_low_s32(b0.val[0]), vget_low_s32(b2.val[0]));
a2.val[0] = vcombine_s32(vget_high_s32(b0.val[0]), vget_high_s32(b2.val[0]));
a1.val[0] = vcombine_s32(vget_low_s32(b0.val[1]), vget_low_s32(b2.val[1]));
a3.val[0] = vcombine_s32(vget_high_s32(b0.val[1]), vget_high_s32(b2.val[1]));
a0.val[1] = vcombine_s32(vget_low_s32(b1.val[0]), vget_low_s32(b3.val[0]));
a2.val[1] = vcombine_s32(vget_high_s32(b1.val[0]), vget_high_s32(b3.val[0]));
a1.val[1] = vcombine_s32(vget_low_s32(b1.val[1]), vget_low_s32(b3.val[1]));
a3.val[1] = vcombine_s32(vget_high_s32(b1.val[1]), vget_high_s32(b3.val[1]));
o0_2 = vsubq_s32(a0.val[0], a3.val[1]); /*B0 - B7*/
o1_2 = vsubq_s32(a1.val[0], a2.val[1]); /*B1 - B6*/
o2_2 = vsubq_s32(a2.val[0], a1.val[1]); /*B2 - B5*/
o3_2 = vsubq_s32(a3.val[0], a0.val[1]); /*B3 - B4*/
e3_2 = vaddq_s32(a3.val[0], a0.val[1]); /*B3 + B4*/
e2_2 = vaddq_s32(a2.val[0], a1.val[1]); /*B2 + B5*/
e1_2 = vaddq_s32(a1.val[0], a2.val[1]); /*B1 + B6*/
e0_2 = vaddq_s32(a0.val[0], a3.val[1]); /*B0 + B7*/
eo1_2 = vsubq_s32(e1_2, e2_2); /*B1 - B2 - B5 + B6*/
ee1_2 = vaddq_s32(e1_2, e2_2); /*B1 + B2 + B5 + B6*/
eo0_2 = vsubq_s32(e0_2, e3_2); /*B0 - B3 - B4 + B7*/
ee0_2 = vaddq_s32(e0_2, e3_2); /*B0 + B3 + B4 + B7*/
/* F4 = B0 - B1 - B2 + B3 + B4 - B5 - B6 + B7*/
h4 = vsubq_s32(ee0_2, ee1_2);
/* F0 = B0 + B1 + B2 + B3 + B4 + B5 + B6 + B7*/
h0 = vaddq_s32(ee0_2, ee1_2);
/* Truncating last 11 bits in H0*/
row0 = vrshrn_n_s32(h0, 5);
/*First half-row of row 1 of transform stage 2 (H0) stored*/
vst1_s16(pi2_dst, row0);
/* Truncating last 11 bits in H4*/
row4 = vrshrn_n_s32(h4, 5);
/*First half-row of row 5 of transform stage 2 (H4) stored*/
vst1_s16(pi2_dst + 4 * dst_strd, row4);
/* F6 = 36*(B0 - B3 - B4 + B7) */
h6 = vmulq_n_s32(eo0_2, 36);
/* F2 = 83*(B0 - B3 - B4 + B7) */
h2 = vmulq_n_s32(eo0_2, 83);
/*H2 = 83*(B0 - B3 - B4 + B7) + 36*(B1 - B2 - B5 + B6)*/
h2 = vmlaq_n_s32(h2, eo1_2, 36);
/*H6 = 36*(B0 - B3 - B4 + B7) - 83*(B1 - B2 - B5 + B6)*/
h6 = vmlsq_n_s32(h6, eo1_2, 83);
/* Truncating last 11 bits in H6*/
row6 = vrshrn_n_s32(h6, 11);
/*First half-row of row 7 of transform stage 2 (H6) stored*/
vst1_s16(pi2_dst + 6 * dst_strd, row6);
/* Truncating last 11 bits in H2*/
row2 = vrshrn_n_s32(h2, 11);
/*First half-row of row 3 of transform stage 2 (H2) stored*/
vst1_s16(pi2_dst + 2 * dst_strd, row2);
h1 = vmulq_n_s32(o0_2, 89); /* H1 = 89*(B0 - B7) */
h3 = vmulq_n_s32(o0_2, 75); /* H3 = 75*(B0 - B7) */
h5 = vmulq_n_s32(o0_2, 50); /* H5 = 50*(B0 - B7) */
h7 = vmulq_n_s32(o0_2, 18); /* H7 = 18*(B0 - B7) */
h7 = vmlsq_n_s32(h7, o1_2, 50); /* H7 = 18*(B0 - B7) - 50*(B1 - B6) */
h5 = vmlsq_n_s32(h5, o1_2, 89); /* H5 = 50*(B0 - B7) - 89*(B1 - B6) */
h3 = vmlsq_n_s32(h3, o1_2, 18); /* H3 = 75*(B0 - B7) - 18*(B1 - B6) */
h1 = vmlaq_n_s32(h1, o1_2, 75); /* H1 = 89*(B0 - B7) + 75*(B1 - B6) */
/* H1 = 89*(B0 - B7) + 75*(B1 - B6) + 50*(B2 - B5) */
h1 = vmlaq_n_s32(h1, o2_2, 50);
/* H3 = 75*(B0 - B7) - 18*(B1 - B6) - 89*(B2 - B5) */
h3 = vmlsq_n_s32(h3, o2_2, 89);
/* H5 = 50*(B0 - B7) - 89*(B1 - B6) + 18*(B2 - B5) */
h5 = vmlaq_n_s32(h5, o2_2, 18);
/* H7 = 18*(B0 - B7) - 50*(B1 - B6) + 75*(B2 - B5) */
h7 = vmlaq_n_s32(h7, o2_2, 75);
/* H7 = 18*(B0 - B7) - 50*(B1 - B6) + 75*(B2 - B5) - 89*(B3 - B4) */
h7 = vmlsq_n_s32(h7, o3_2, 89);
/* Truncating last 11 bits in H7*/
row7 = vrshrn_n_s32(h7, 11);
/*First half-row of row 8 of transform stage 2 (H7) stored*/
vst1_s16(pi2_dst + 7 * dst_strd, row7);
/* H5 = 50*(B0 - B7) - 89*(B1 - B6) + 18*(B2 - B5) + 75*(B3 - B4) */
h5 = vmlaq_n_s32(h5, o3_2, 75);
/* Truncating last 11 bits in H5*/
row5 = vrshrn_n_s32(h5, 11);
/*First half-row of row 6 of transform stage 2 (H5) stored*/
vst1_s16(pi2_dst + 5 * dst_strd, row5);
/* H3 = 75*(B0 - B7) - 18*(B1 - B6) - 89*(B2 - B5) - 50*(B3 - B4) */
h3 = vmlsq_n_s32(h3, o3_2, 50);
/* Truncating last 11 bits in H3*/
row3 = vrshrn_n_s32(h3, 11);
/*First half-row of row 4 of transform stage 2 (H3) stored*/
vst1_s16(pi2_dst + 3 * dst_strd, row3);
/* H1 = 89*(B0 - B7) + 75*(B1 - B6) + 50*(B2 - B5) + 18*(B3 - B4) */
h1 = vmlaq_n_s32(h1, o3_2, 18);
/* Truncating last 11 bits in H1*/
row1 = vrshrn_n_s32(h1, 11);
/*First half-row of row 2 of transform stage 2 (H1) stored*/
vst1_s16(pi2_dst + dst_strd, row1);
}
pi2_dst += 4;
{
int32x4_t h0, h1, h2, h3, h4, h5, h6, h7;
int32x4_t e0_2, e1_2, e2_2, e3_2;
int32x4_t o0_2, o1_2, o2_2, o3_2;
int32x4_t ee1_2, eo1_2, eo0_2, ee0_2;
int16x4_t row0, row1, row2, row3, row4, row5, row6, row7;
/*Transposing second half of transform stage 1 (1)*/
int32x4x2_t b1 = vtrnq_s32(a4.val[1], a5.val[1]);
int32x4x2_t b3 = vtrnq_s32(a6.val[1], a7.val[1]);
int32x4x2_t b0 = vtrnq_s32(a4.val[0], a5.val[0]);
int32x4x2_t b2 = vtrnq_s32(a6.val[0], a7.val[0]);
/*Transposing second half of transform stage 1 (2)*/
a0.val[0] = vcombine_s32(vget_low_s32(b0.val[0]), vget_low_s32(b2.val[0]));
a2.val[0] = vcombine_s32(vget_high_s32(b0.val[0]), vget_high_s32(b2.val[0]));
a1.val[0] = vcombine_s32(vget_low_s32(b0.val[1]), vget_low_s32(b2.val[1]));
a3.val[0] = vcombine_s32(vget_high_s32(b0.val[1]), vget_high_s32(b2.val[1]));
a0.val[1] = vcombine_s32(vget_low_s32(b1.val[0]), vget_low_s32(b3.val[0]));
a2.val[1] = vcombine_s32(vget_high_s32(b1.val[0]), vget_high_s32(b3.val[0]));
a1.val[1] = vcombine_s32(vget_low_s32(b1.val[1]), vget_low_s32(b3.val[1]));
a3.val[1] = vcombine_s32(vget_high_s32(b1.val[1]), vget_high_s32(b3.val[1]));
o0_2 = vsubq_s32(a0.val[0], a3.val[1]); /*B0 - B7*/
o1_2 = vsubq_s32(a1.val[0], a2.val[1]); /*B1 - B6*/
o2_2 = vsubq_s32(a2.val[0], a1.val[1]); /*B2 - B5*/
o3_2 = vsubq_s32(a3.val[0], a0.val[1]); /*B3 - B4*/
e3_2 = vaddq_s32(a3.val[0], a0.val[1]); /*B3 + B4*/
e2_2 = vaddq_s32(a2.val[0], a1.val[1]); /*B2 + B5*/
e1_2 = vaddq_s32(a1.val[0], a2.val[1]); /*B1 + B6*/
e0_2 = vaddq_s32(a0.val[0], a3.val[1]); /*B0 + B7*/
eo1_2 = vsubq_s32(e1_2, e2_2); /*B1 - B2 - B5 + B6*/
ee1_2 = vaddq_s32(e1_2, e2_2); /*B1 + B2 + B5 + B6*/
eo0_2 = vsubq_s32(e0_2, e3_2); /*B0 - B3 - B4 + B7*/
ee0_2 = vaddq_s32(e0_2, e3_2); /*B0 + B3 + B4 + B7*/
/* F4 = B0 - B1 - B2 + B3 + B4 - B5 - B6 + B7*/
h4 = vsubq_s32(ee0_2, ee1_2);
/* F0 = B0 + B1 + B2 + B3 + B4 + B5 + B6 + B7*/
h0 = vaddq_s32(ee0_2, ee1_2);
/* Truncating last 11 bits in H0*/
row0 = vrshrn_n_s32(h0, 5);
/*First half-row of row 1 of transform stage 2 (H0) stored*/
vst1_s16(pi2_dst, row0);
/* Truncating last 11 bits in H4*/
row4 = vrshrn_n_s32(h4, 5);
/*First half-row of row 5 of transform stage 2 (H4) stored*/
vst1_s16(pi2_dst + 4 * dst_strd, row4);
/* F6 = 36*(B0 - B3 - B4 + B7) */
h6 = vmulq_n_s32(eo0_2, 36);
/* F2 = 83*(B0 - B3 - B4 + B7) */
h2 = vmulq_n_s32(eo0_2, 83);
/*H2 = 83*(B0 - B3 - B4 + B7) + 36*(B1 - B2 - B5 + B6)*/
h2 = vmlaq_n_s32(h2, eo1_2, 36);
/*H6 = 36*(B0 - B3 - B4 + B7) - 83*(B1 - B2 - B5 + B6)*/
h6 = vmlsq_n_s32(h6, eo1_2, 83);
/* Truncating last 11 bits in H6*/
row6 = vrshrn_n_s32(h6, 11);
/*First half-row of row 7 of transform stage 2 (H6) stored*/
vst1_s16(pi2_dst + 6 * dst_strd, row6);
/* Truncating last 11 bits in H2*/
row2 = vrshrn_n_s32(h2, 11);
/*First half-row of row 3 of transform stage 2 (H2) stored*/
vst1_s16(pi2_dst + 2 * dst_strd, row2);
h1 = vmulq_n_s32(o0_2, 89); /* H1 = 89*(B0 - B7) */
h3 = vmulq_n_s32(o0_2, 75); /* H3 = 75*(B0 - B7) */
h5 = vmulq_n_s32(o0_2, 50); /* H5 = 50*(B0 - B7) */
h7 = vmulq_n_s32(o0_2, 18); /* H7 = 18*(B0 - B7) */
h7 = vmlsq_n_s32(h7, o1_2, 50); /* H7 = 18*(B0 - B7) - 50*(B1 - B6) */
h5 = vmlsq_n_s32(h5, o1_2, 89); /* H5 = 50*(B0 - B7) - 89*(B1 - B6) */
h3 = vmlsq_n_s32(h3, o1_2, 18); /* H3 = 75*(B0 - B7) - 18*(B1 - B6) */
h1 = vmlaq_n_s32(h1, o1_2, 75); /* H1 = 89*(B0 - B7) + 75*(B1 - B6) */
/* H1 = 89*(B0 - B7) + 75*(B1 - B6) + 50*(B2 - B5) */
h1 = vmlaq_n_s32(h1, o2_2, 50);
/* H3 = 75*(B0 - B7) - 18*(B1 - B6) - 89*(B2 - B5) */
h3 = vmlsq_n_s32(h3, o2_2, 89);
/* H5 = 50*(B0 - B7) - 89*(B1 - B6) + 18*(B2 - B5) */
h5 = vmlaq_n_s32(h5, o2_2, 18);
/* H7 = 18*(B0 - B7) - 50*(B1 - B6) + 75*(B2 - B5) */
h7 = vmlaq_n_s32(h7, o2_2, 75);
/* H7 = 18*(B0 - B7) - 50*(B1 - B6) + 75*(B2 - B5) - 89*(B3 - B4) */
h7 = vmlsq_n_s32(h7, o3_2, 89);
/* Truncating last 11 bits in H7*/
row7 = vrshrn_n_s32(h7, 11);
/*First half-row of row 8 of transform stage 2 (H7) stored*/
vst1_s16(pi2_dst + 7 * dst_strd, row7);
/* H5 = 50*(B0 - B7) - 89*(B1 - B6) + 18*(B2 - B5) + 75*(B3 - B4) */
h5 = vmlaq_n_s32(h5, o3_2, 75);
/* Truncating last 11 bits in H5*/
row5 = vrshrn_n_s32(h5, 11);
/*First half-row of row 6 of transform stage 2 (H5) stored*/
vst1_s16(pi2_dst + 5 * dst_strd, row5);
/* H3 = 75*(B0 - B7) - 18*(B1 - B6) - 89*(B2 - B5) - 50*(B3 - B4) */
h3 = vmlsq_n_s32(h3, o3_2, 50);
/* Truncating last 11 bits in H3*/
row3 = vrshrn_n_s32(h3, 11);
/*First half-row of row 4 of transform stage 2 (H3) stored*/
vst1_s16(pi2_dst + 3 * dst_strd, row3);
/* H1 = 89*(B0 - B7) + 75*(B1 - B6) + 50*(B2 - B5) + 18*(B3 - B4) */
h1 = vmlaq_n_s32(h1, o3_2, 18);
/* Truncating last 11 bits in H1*/
row1 = vrshrn_n_s32(h1, 11);
/*First half-row of row 2 of transform stage 2 (H1) stored*/
vst1_s16(pi2_dst + dst_strd, row1);
}
return sad;
}
static INLINE void load(const uint8_t *a, int stride, uint8x8_t *b,
CHROMA_PLANE_ID_T e_chroma_plane)
{
int i;
if(e_chroma_plane == NULL_PLANE)
{
for (i = 0; i < 16; i++)
{
b[i] = vld1_u8(a);
a += stride;
}
}
else
{
for (i = 0; i < 16; i++)
{
b[i] = vld2_u8(a).val[e_chroma_plane];
a += stride;
}
}
}
// Store 8 16x8 values, assuming stride == 16.
static INLINE void store(WORD16 *a, int16x8_t *b /*[8]*/)
{
int i;
for (i = 0; i < 8; i++)
{
vst1q_s16(a, b[i]);
a += 16;
}
}
static INLINE void cross_input_16(int16x8_t *a /*[16]*/, int16x8_t *b /*[16]*/)
{
b[0] = vaddq_s16(a[0], a[15]);
b[1] = vaddq_s16(a[1], a[14]);
b[2] = vaddq_s16(a[2], a[13]);
b[3] = vaddq_s16(a[3], a[12]);
b[4] = vaddq_s16(a[4], a[11]);
b[5] = vaddq_s16(a[5], a[10]);
b[6] = vaddq_s16(a[6], a[9]);
b[7] = vaddq_s16(a[7], a[8]);
b[8] = vsubq_s16(a[7], a[8]);
b[9] = vsubq_s16(a[6], a[9]);
b[10] = vsubq_s16(a[5], a[10]);
b[11] = vsubq_s16(a[4], a[11]);
b[12] = vsubq_s16(a[3], a[12]);
b[13] = vsubq_s16(a[2], a[13]);
b[14] = vsubq_s16(a[1], a[14]);
b[15] = vsubq_s16(a[0], a[15]);
}
static INLINE void cross_input_32(int32x4x2_t *a /*[16][2]*/, int32x4x2_t *b /*[16][2]*/)
{
WORD32 i;
for(i = 0; i < 2; i++)
{
b[0].val[i] = vaddq_s32(a[0].val[i], a[15].val[i]);
b[1].val[i] = vaddq_s32(a[1].val[i], a[14].val[i]);
b[2].val[i] = vaddq_s32(a[2].val[i], a[13].val[i]);
b[3].val[i] = vaddq_s32(a[3].val[i], a[12].val[i]);
b[4].val[i] = vaddq_s32(a[4].val[i], a[11].val[i]);
b[5].val[i] = vaddq_s32(a[5].val[i], a[10].val[i]);
b[6].val[i] = vaddq_s32(a[6].val[i], a[9].val[i]);
b[7].val[i] = vaddq_s32(a[7].val[i], a[8].val[i]);
b[8].val[i] = vsubq_s32(a[7].val[i], a[8].val[i]);
b[9].val[i] = vsubq_s32(a[6].val[i], a[9].val[i]);
b[10].val[i] = vsubq_s32(a[5].val[i], a[10].val[i]);
b[11].val[i] = vsubq_s32(a[4].val[i], a[11].val[i]);
b[12].val[i] = vsubq_s32(a[3].val[i], a[12].val[i]);
b[13].val[i] = vsubq_s32(a[2].val[i], a[13].val[i]);
b[14].val[i] = vsubq_s32(a[1].val[i], a[14].val[i]);
b[15].val[i] = vsubq_s32(a[0].val[i], a[15].val[i]);
}
}
static INLINE int32x4_t diff(uint8x8_t *a /*[16]*/, uint8x8_t *b /*[16]*/, int16x8_t *c /*[16]*/)
{
int i;
int16x8_t abs = vdupq_n_s16(0);
for (i = 0; i < 16; i++)
{
c[i] = vreinterpretq_s16_u16(vsubl_u8(a[i], b[i]));
abs = vaddq_s16(abs, vabsq_s16(c[i]));
}
return vpaddlq_s16(abs);
}
static INLINE void partial_round_shift(int32x4x2_t *a, int16x8_t *b /*[16]*/)
{
WORD32 shift = 13, add;
add = 1 << (shift - 1);
const int32x4_t vecadd = vdupq_n_s32(add);
b[0] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[0].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[0].val[1], vecadd), 13));
b[1] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[1].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[1].val[1], vecadd), 13));
b[2] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[2].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[2].val[1], vecadd), 13));
b[3] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[3].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[3].val[1], vecadd), 13));
b[4] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[4].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[4].val[1], vecadd), 13));
b[5] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[5].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[5].val[1], vecadd), 13));
b[6] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[6].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[6].val[1], vecadd), 13));
b[7] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[7].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[7].val[1], vecadd), 13));
b[8] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[8].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[8].val[1], vecadd), 13));
b[9] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[9].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[9].val[1], vecadd), 13));
b[10] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[10].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[10].val[1], vecadd), 13));
b[11] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[11].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[11].val[1], vecadd), 13));
b[12] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[12].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[12].val[1], vecadd), 13));
b[13] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[13].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[13].val[1], vecadd), 13));
b[14] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[14].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[14].val[1], vecadd), 13));
b[15] = vcombine_s16(
vshrn_n_s32(vaddq_s32(a[15].val[0], vecadd), 13),
vshrn_n_s32(vaddq_s32(a[15].val[1], vecadd), 13));
}
static INLINE int32x4_t
add4(int32x4_t row1_low, int32x4_t row1_high, int32x4_t row2_low, int32x4_t row2_high)
{
int32x4_t sum1, sum2;
sum1 = vaddq_s32(row1_low, row1_high);
sum2 = vaddq_s32(row2_low, row2_high);
return vaddq_s32(sum1, sum2);
}
static INLINE void butterfly_one_coeff_16_32(
int16x8_t a, int16x8_t b, int16_t c, int32x4x2_t *row1, int32x4x2_t *row2)
{
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c);
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c);
//printf("multiply done\n");
row1->val[0] = vmlal_n_s16(a0, vget_low_s16(b), c);
row1->val[1] = vmlal_n_s16(a1, vget_high_s16(b), c);
row2->val[0] = vmlsl_n_s16(a0, vget_low_s16(b), c);
row2->val[1] = vmlsl_n_s16(a1, vget_high_s16(b), c);
}
static INLINE void butterfly_two_coeff_16_32(
int16x8_t a, int16x8_t b, int16_t c0, int16_t c1, int32x4x2_t *row1, int32x4x2_t *row2)
{
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0);
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0);
const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1);
const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1);
row1->val[0] = vmlal_n_s16(a2, vget_low_s16(b), c0);
row1->val[1] = vmlal_n_s16(a3, vget_high_s16(b), c0);
row2->val[0] = vmlsl_n_s16(a0, vget_low_s16(b), c1);
row2->val[1] = vmlsl_n_s16(a1, vget_high_s16(b), c1);
}
static INLINE void butterfly_one_coeff_32_32(
int32x4x2_t a, int32x4x2_t b, int32_t c, int32x4x2_t *row1, int32x4x2_t *row2)
{
const int32x4_t a0 = vmulq_n_s32(a.val[0], c);
const int32x4_t a1 = vmulq_n_s32(a.val[1], c);
row1->val[0] = vmlaq_n_s32(a0, b.val[0], c);
row1->val[1] = vmlaq_n_s32(a1, b.val[1], c);
row2->val[0] = vmlsq_n_s32(a0, b.val[0], c);
row2->val[1] = vmlsq_n_s32(a1, b.val[1], c);
}
static INLINE void butterfly_two_coeff_32_32(
int32x4x2_t a, int32x4x2_t b, int32_t c0, int32_t c1, int32x4x2_t *row1, int32x4x2_t *row2)
{
const int32x4_t a0 = vmulq_n_s32(a.val[0], c0);
const int32x4_t a1 = vmulq_n_s32(a.val[1], c0);
const int32x4_t a2 = vmulq_n_s32(a.val[0], c1);
const int32x4_t a3 = vmulq_n_s32(a.val[1], c1);
row1->val[0] = vmlaq_n_s32(a2, b.val[0], c0);
row1->val[1] = vmlaq_n_s32(a3, b.val[1], c0);
row2->val[0] = vmlsq_n_s32(a0, b.val[0], c1);
row2->val[1] = vmlsq_n_s32(a1, b.val[1], c1);
}
// Transpose 8x8 to a new location. Don't use transpose_neon.h because those
// are all in-place.
static INLINE void transpose_8x8(int32x4x2_t *a /*[8][2]*/, int32x4x2_t *b)
{
const int32x4x2_t c0 = vtrnq_s32(a[0].val[0], a[1].val[0]);
const int32x4x2_t c1 = vtrnq_s32(a[2].val[0], a[3].val[0]);
const int32x4x2_t c2 = vtrnq_s32(a[4].val[0], a[5].val[0]);
const int32x4x2_t c3 = vtrnq_s32(a[6].val[0], a[7].val[0]);
const int32x4x2_t c4 = vtrnq_s32(a[0].val[1], a[1].val[1]);
const int32x4x2_t c5 = vtrnq_s32(a[2].val[1], a[3].val[1]);
const int32x4x2_t c6 = vtrnq_s32(a[4].val[1], a[5].val[1]);
const int32x4x2_t c7 = vtrnq_s32(a[6].val[1], a[7].val[1]);
const int32x4x2_t d0 = vtrnq_s64_to_s32(c0.val[0], c1.val[0]);
const int32x4x2_t d1 = vtrnq_s64_to_s32(c0.val[1], c1.val[1]);
const int32x4x2_t d2 = vtrnq_s64_to_s32(c2.val[0], c3.val[0]);
const int32x4x2_t d3 = vtrnq_s64_to_s32(c2.val[1], c3.val[1]);
const int32x4x2_t d4 = vtrnq_s64_to_s32(c4.val[0], c5.val[0]);
const int32x4x2_t d5 = vtrnq_s64_to_s32(c4.val[1], c5.val[1]);
const int32x4x2_t d6 = vtrnq_s64_to_s32(c6.val[0], c7.val[0]);
const int32x4x2_t d7 = vtrnq_s64_to_s32(c6.val[1], c7.val[1]);
b[0].val[0] = d0.val[0];
b[0].val[1] = d2.val[0];
b[1].val[0] = d1.val[0];
b[1].val[1] = d3.val[0];
b[2].val[0] = d0.val[1];
b[2].val[1] = d2.val[1];
b[3].val[0] = d1.val[1];
b[3].val[1] = d3.val[1];
b[4].val[0] = d4.val[0];
b[4].val[1] = d6.val[0];
b[5].val[0] = d5.val[0];
b[5].val[1] = d7.val[0];
b[6].val[0] = d4.val[1];
b[6].val[1] = d6.val[1];
b[7].val[0] = d5.val[1];
b[7].val[1] = d7.val[1];
}
static void dct_body_16_32(int16x8_t *in /*[16]*/, int32x4x2_t *out /*[16]*/)
{
int16x8_t s[8];
int16x8_t x[4];
int32x4x2_t tmp0, tmp1, tmp2, tmp3;
int32x4x2_t tmp4, tmp5, tmp6, tmp7;
s[0] = vaddq_s16(in[0], in[7]);
s[1] = vaddq_s16(in[1], in[6]);
s[2] = vaddq_s16(in[2], in[5]);
s[3] = vaddq_s16(in[3], in[4]);
s[4] = vsubq_s16(in[3], in[4]);
s[5] = vsubq_s16(in[2], in[5]);
s[6] = vsubq_s16(in[1], in[6]);
s[7] = vsubq_s16(in[0], in[7]);
x[0] = vaddq_s16(s[0], s[3]);
x[1] = vaddq_s16(s[1], s[2]);
x[2] = vsubq_s16(s[1], s[2]);
x[3] = vsubq_s16(s[0], s[3]);
// Type 1
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
butterfly_one_coeff_16_32(x[0], x[1], 64, &out[0], &out[8]);
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
butterfly_two_coeff_16_32(x[3], x[2], 36, 83, &out[4], &out[12]);
// Type 2
butterfly_two_coeff_16_32(s[7], s[4], 18, 89, &tmp0, &tmp1);
butterfly_two_coeff_16_32(s[5], s[6], 75, 50, &tmp2, &tmp3);
out[2].val[0] = vaddq_s32(tmp0.val[0], tmp2.val[0]);
out[2].val[1] = vaddq_s32(tmp0.val[1], tmp2.val[1]);
out[14].val[0] = vaddq_s32(tmp1.val[0], tmp3.val[0]);
out[14].val[1] = vaddq_s32(tmp1.val[1], tmp3.val[1]);
butterfly_two_coeff_16_32(s[7], s[4], 75, 50, &tmp0, &tmp1);
butterfly_two_coeff_16_32(s[5], s[6], -89, 18, &tmp2, &tmp3);
out[10].val[0] = vaddq_s32(tmp0.val[0], tmp2.val[0]);
out[10].val[1] = vaddq_s32(tmp0.val[1], tmp2.val[1]);
out[6].val[0] = vaddq_s32(tmp1.val[0], tmp3.val[0]);
out[6].val[1] = vaddq_s32(tmp1.val[1], tmp3.val[1]);
// Type 3
butterfly_two_coeff_16_32(in[8], in[15], 9, -90, &tmp0, &tmp1);
butterfly_two_coeff_16_32(in[9], in[14], 87, 25, &tmp2, &tmp3);
butterfly_two_coeff_16_32(in[10], in[13], 43, -80, &tmp4, &tmp5);
butterfly_two_coeff_16_32(in[11], in[12], 70, 57, &tmp6, &tmp7);
out[1].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
out[1].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
out[15].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
out[15].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
butterfly_two_coeff_16_32(in[8], in[15], 87, -25, &tmp0, &tmp1);
butterfly_two_coeff_16_32(in[9], in[14], -70, -57, &tmp2, &tmp3);
butterfly_two_coeff_16_32(in[10], in[13], 9, -90, &tmp4, &tmp5);
butterfly_two_coeff_16_32(in[11], in[12], -80, 43, &tmp6, &tmp7);
out[3].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
out[3].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
out[13].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
out[13].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
butterfly_two_coeff_16_32(in[8], in[15], 43, -80, &tmp0, &tmp1);
butterfly_two_coeff_16_32(in[9], in[14], 9, 90, &tmp2, &tmp3);
butterfly_two_coeff_16_32(in[10], in[13], 57, 70, &tmp4, &tmp5);
butterfly_two_coeff_16_32(in[11], in[12], -87, -25, &tmp6, &tmp7);
out[5].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
out[5].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
out[11].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
out[11].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
butterfly_two_coeff_16_32(in[8], in[15], 70, -57, &tmp0, &tmp1);
butterfly_two_coeff_16_32(in[9], in[14], -80, 43, &tmp2, &tmp3);
butterfly_two_coeff_16_32(in[10], in[13], -87, 25, &tmp4, &tmp5);
butterfly_two_coeff_16_32(in[11], in[12], 90, -9, &tmp6, &tmp7);
out[7].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
out[7].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
out[9].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
out[9].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
}
static void dct_body_32_32(int32x4x2_t *in /*[16]*/, int32x4x2_t *out /*[16]*/)
{
int32x4x2_t s[8];
int32x4x2_t x[4];
int32x4x2_t tmp0, tmp1, tmp2, tmp3;
int32x4x2_t tmp4, tmp5, tmp6, tmp7;
WORD32 i;
for(i = 0; i < 2; i++)
{
s[0].val[i] = vaddq_s32(in[0].val[i], in[7].val[i]);
s[1].val[i] = vaddq_s32(in[1].val[i], in[6].val[i]);
s[2].val[i] = vaddq_s32(in[2].val[i], in[5].val[i]);
s[3].val[i] = vaddq_s32(in[3].val[i], in[4].val[i]);
s[4].val[i] = vsubq_s32(in[3].val[i], in[4].val[i]);
s[5].val[i] = vsubq_s32(in[2].val[i], in[5].val[i]);
s[6].val[i] = vsubq_s32(in[1].val[i], in[6].val[i]);
s[7].val[i] = vsubq_s32(in[0].val[i], in[7].val[i]);
x[0].val[i] = vaddq_s32(s[0].val[i], s[3].val[i]);
x[1].val[i] = vaddq_s32(s[1].val[i], s[2].val[i]);
x[2].val[i] = vsubq_s32(s[1].val[i], s[2].val[i]);
x[3].val[i] = vsubq_s32(s[0].val[i], s[3].val[i]);
}
// Type 1
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
butterfly_one_coeff_32_32(x[0], x[1], 64, &out[0], &out[8]);
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
butterfly_two_coeff_32_32(x[3], x[2], 36, 83, &out[4], &out[12]);
// Type 2
butterfly_two_coeff_32_32(s[7], s[4], 18, 89, &tmp0, &tmp1);
butterfly_two_coeff_32_32(s[5], s[6], 75, 50, &tmp2, &tmp3);
out[2].val[0] = vaddq_s32(tmp0.val[0], tmp2.val[0]);
out[2].val[1] = vaddq_s32(tmp0.val[1], tmp2.val[1]);
out[14].val[0] = vaddq_s32(tmp1.val[0], tmp3.val[0]);
out[14].val[1] = vaddq_s32(tmp1.val[1], tmp3.val[1]);
butterfly_two_coeff_32_32(s[7], s[4], 75, 50, &tmp0, &tmp1);
butterfly_two_coeff_32_32(s[5], s[6], -89, 18, &tmp2, &tmp3);
out[10].val[0] = vaddq_s32(tmp0.val[0], tmp2.val[0]);
out[10].val[1] = vaddq_s32(tmp0.val[1], tmp2.val[1]);
out[6].val[0] = vaddq_s32(tmp1.val[0], tmp3.val[0]);
out[6].val[1] = vaddq_s32(tmp1.val[1], tmp3.val[1]);
// Type 3
butterfly_two_coeff_32_32(in[8], in[15], 9, -90, &tmp0, &tmp1);
butterfly_two_coeff_32_32(in[9], in[14], 87, 25, &tmp2, &tmp3);
butterfly_two_coeff_32_32(in[10], in[13], 43, -80, &tmp4, &tmp5);
butterfly_two_coeff_32_32(in[11], in[12], 70, 57, &tmp6, &tmp7);
out[1].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
out[1].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
out[15].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
out[15].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
butterfly_two_coeff_32_32(in[8], in[15], 87, -25, &tmp0, &tmp1);
butterfly_two_coeff_32_32(in[9], in[14], -70, -57, &tmp2, &tmp3);
butterfly_two_coeff_32_32(in[10], in[13], 9, -90, &tmp4, &tmp5);
butterfly_two_coeff_32_32(in[11], in[12], -80, 43, &tmp6, &tmp7);
out[3].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
out[3].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
out[13].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
out[13].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
butterfly_two_coeff_32_32(in[8], in[15], 43, -80, &tmp0, &tmp1);
butterfly_two_coeff_32_32(in[9], in[14], 9, 90, &tmp2, &tmp3);
butterfly_two_coeff_32_32(in[10], in[13], 57, 70, &tmp4, &tmp5);
butterfly_two_coeff_32_32(in[11], in[12], -87, -25, &tmp6, &tmp7);
out[5].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
out[5].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
out[11].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
out[11].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
butterfly_two_coeff_32_32(in[8], in[15], 70, -57, &tmp0, &tmp1);
butterfly_two_coeff_32_32(in[9], in[14], -80, 43, &tmp2, &tmp3);
butterfly_two_coeff_32_32(in[10], in[13], -87, 25, &tmp4, &tmp5);
butterfly_two_coeff_32_32(in[11], in[12], 90, -9, &tmp6, &tmp7);
out[7].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
out[7].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
out[9].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
out[9].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
}
/**
*******************************************************************************
*
* @brief
* This function performs residue calculation and forward transform on
* input pixels
*
* @par Description:
* Performs residue calculation by subtracting source and prediction and
* followed by forward transform
*
* @param[in] pu1_src
* Input 16x16 pixels
*
* @param[in] pu1_pred
* Prediction data
*
* @param[in] pi2_tmp
* Temporary buffer of size 16x16
*
* @param[out] pi2_dst
* Output 16x16 coefficients
*
* @param[in] src_strd
* Input stride
*
* @param[in] pred_strd
* Prediction Stride
*
* @param[in] dst_strd
* Output Stride
*
* @param[in] e_chroma_plane
* Enum singalling chroma plane
*
* @returns Void
*
* @remarks
* None
*
*******************************************************************************
*/
UWORD32 ihevc_resi_trans_16x16_neon(
UWORD8 *pu1_src,
UWORD8 *pu1_pred,
WORD32 *pi4_temp,
WORD16 *pi2_dst,
WORD32 src_strd,
WORD32 pred_strd,
WORD32 dst_strd,
CHROMA_PLANE_ID_T e_chroma_plane)
{
UWORD32 u4_blk_sad = 0;
WORD32 chroma_flag;
uint8x8_t temp0[16], temp1[16];
int16x8_t temp2[16], temp3[16];
int32x4_t tmp_a, tmp_b;
int64x2_t tmp_c;
int32x2_t sad_v;
int32x4x2_t out0[16], out1[16], temp4[16], temp5[16];
(void)pi4_temp;
chroma_flag = e_chroma_plane != NULL_PLANE;
/* Residue + Forward Transform 1st stage */
// Left half.
load(pu1_src, src_strd, temp0, e_chroma_plane);
load(pu1_pred, pred_strd, temp1, e_chroma_plane);
tmp_a = diff(temp0, temp1, temp2);
cross_input_16(temp2, temp3);
dct_body_16_32(temp3, out0);
// Right half.
load(pu1_src + 8 * (1 + chroma_flag), src_strd, temp0, e_chroma_plane);
load(pu1_pred + 8 * (1 + chroma_flag), pred_strd, temp1, e_chroma_plane);
tmp_b = diff(temp0, temp1, temp2);
cross_input_16(temp2, temp3);
dct_body_16_32(temp3, out1);
tmp_a = vaddq_s32(tmp_a, tmp_b);
tmp_c = vpaddlq_s32(tmp_a);
sad_v = vadd_s32(vreinterpret_s32_s64(vget_low_s64(tmp_c)),
vreinterpret_s32_s64(vget_high_s64(tmp_c)));
u4_blk_sad = vget_lane_s32(sad_v, 0);
// Transpose top left and top right quarters into one contiguous location to
// process to the top half.
transpose_8x8(&out0[0], &temp4[0]);
transpose_8x8(&out1[0], &temp4[8]);
cross_input_32(temp4, temp5);
dct_body_32_32(temp5, temp4);
partial_round_shift(temp4, temp2);
transpose_s16_8x8(
&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4], &temp2[5], &temp2[6], &temp2[7]);
transpose_s16_8x8(
&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12], &temp2[13], &temp2[14], &temp2[15]);
store(pi2_dst, &temp2[0]);
store(pi2_dst + 8, &temp2[8]);
pi2_dst += 8 * dst_strd;
// Transpose bottom left and bottom right quarters into one contiguous
// location to process to the bottom half.
transpose_8x8(&out0[8], &out1[0]);
transpose_s32_8x8(
&out1[8], &out1[9], &out1[10], &out1[11], &out1[12], &out1[13], &out1[14], &out1[15]);
cross_input_32(out1, temp5);
dct_body_32_32(temp5, temp4);
partial_round_shift(temp4, temp2);
transpose_s16_8x8(
&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4], &temp2[5], &temp2[6], &temp2[7]);
transpose_s16_8x8(
&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12], &temp2[13], &temp2[14], &temp2[15]);
store(pi2_dst, &temp2[0]);
store(pi2_dst + 8, &temp2[8]);
return u4_blk_sad;
}