| /****************************************************************************** |
| * |
| * Copyright (C) 2012 Ittiam Systems Pvt Ltd, Bangalore |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at: |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| ******************************************************************************/ |
| /** |
| ******************************************************************************* |
| * @file |
| * ihevc_sao_atom_intr.c |
| * |
| * @brief |
| * Contains function definitions for Sample adaptive offset(SAO) used in-loop |
| * filtering |
| * |
| * @author |
| * 100592 |
| * |
| * @par List of Functions: |
| * - ihevc_sao_band_offset_luma_ssse3() |
| * - ihevc_sao_band_offset_chroma_ssse3() |
| * - ihevc_sao_edge_offset_class0_ssse3() |
| * - ihevc_sao_edge_offset_class0_chroma_ssse3() |
| * - ihevc_sao_edge_offset_class1_ssse3() |
| * - ihevc_sao_edge_offset_class1_chroma_ssse3() |
| * - ihevc_sao_edge_offset_class2_ssse3() |
| * - ihevc_sao_edge_offset_class2_chroma_ssse3() |
| * - ihevc_sao_edge_offset_class3_ssse3() |
| * - ihevc_sao_edge_offset_class3_chroma_ssse3() |
| * |
| * @remarks |
| * None |
| * |
| ******************************************************************************* |
| */ |
| /*****************************************************************************/ |
| /* File Includes */ |
| /*****************************************************************************/ |
| #include <stdio.h> |
| |
| #include "ihevc_typedefs.h" |
| #include "ihevc_platform_macros.h" |
| #include "ihevc_macros.h" |
| #include "ihevc_func_selector.h" |
| #include "ihevc_defs.h" |
| #include "ihevc_tables_x86_intr.h" |
| #include "ihevc_common_tables.h" |
| #include "ihevc_sao.h" |
| |
| #include <immintrin.h> |
| |
| #define NUM_BAND_TABLE 32 |
| /** |
| ******************************************************************************* |
| * |
| * @brief |
| * Has two sets of functions : band offset and edge offset both for luma and chroma |
| * edge offset has horizontal ,vertical, 135 degree and 45 degree |
| * |
| * @par Description: |
| * |
| * |
| * @param[in-out] pu1_src |
| * Pointer to the source |
| * |
| * @param[in] src_strd |
| * Source stride |
| * |
| * @param[in-out] pu1_src_left |
| * source left boundary |
| * |
| * @param[in-out] pu1_src_top |
| * Source top boundary |
| * |
| * @param[in-out] pu1_src_top_left |
| * Source top left boundary |
| * |
| * @param[in] pu1_src_top_right |
| * Source top right boundary |
| * |
| * @param[in] pu1_src_bot_left |
| * Source bottom left boundary |
| * |
| * @param[in] pu1_avail |
| * boundary availability flags |
| * |
| * @param[in] pi1_sao_offset_u |
| * Chroma U sao offset values |
| * |
| * @param[in] pi1_sao_offset_v |
| * Chroma V sao offset values |
| * |
| * @param[in] pi1_sao_offset |
| * Luma sao offset values |
| * |
| * @param[in] wd |
| * width of the source |
| |
| * @param[in] ht |
| * height of the source |
| * @returns |
| * |
| * @remarks |
| * None |
| * |
| ******************************************************************************* |
| */ |
| |
| |
| void ihevc_sao_band_offset_luma_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| WORD32 sao_band_pos, |
| WORD8 *pi1_sao_offset, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_cpy; |
| WORD32 wd_rem; |
| WORD8 offset = 0; |
| |
| __m128i src_temp0_8x16b, src_temp1_8x16b, src_temp2_8x16b, src_temp3_8x16b; |
| __m128i band_table0_8x16b, band_table1_8x16b, band_table2_8x16b, band_table3_8x16b; |
| __m128i tmp_set_128i_1, tmp_set_128i_2, tmp_set_128i_3, tmp_set_128i_4; |
| __m128i band_pos_16x8b; |
| __m128i sao_offset; |
| __m128i cmp_mask, cmp_store; |
| |
| /* Updating left and top-left and top */ |
| for(row = 0; row < ht; row++) |
| { |
| pu1_src_left[row] = pu1_src[row * src_strd + (wd - 1)]; |
| } |
| pu1_src_top_left[0] = pu1_src_top[wd - 1]; |
| for(col = 0; col < wd; col += 8) |
| { |
| tmp_set_128i_1 = _mm_loadl_epi64((__m128i *)(pu1_src + (ht - 1) * src_strd + offset)); |
| _mm_storel_epi64((__m128i *)(pu1_src_top + offset), tmp_set_128i_1); |
| offset += 8; |
| } |
| |
| //replicating sao_band_pos as 8 bit value 16 times |
| |
| |
| band_pos_16x8b = _mm_set1_epi16((WORD16)(sao_band_pos << 3)); |
| //value set for sao_offset extraction |
| tmp_set_128i_1 = _mm_set_epi8(128, 1, 128, 1, 128, 1, 128, 1, 128, 1, 128, 1, 128, 1, 128, 1); |
| tmp_set_128i_2 = _mm_set_epi8(128, 2, 128, 2, 128, 2, 128, 2, 128, 2, 128, 2, 128, 2, 128, 2); |
| tmp_set_128i_3 = _mm_set_epi8(128, 3, 128, 3, 128, 3, 128, 3, 128, 3, 128, 3, 128, 3, 128, 3); |
| tmp_set_128i_4 = _mm_set_epi8(128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4); |
| |
| //loaded sao offset values |
| sao_offset = _mm_loadl_epi64((__m128i *)pi1_sao_offset); |
| |
| //loading 16bit 32values of gu2_table_band_idx consecutively in 4 registers |
| band_table0_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx)); |
| band_table1_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 8)); |
| band_table2_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 16)); |
| band_table3_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 24)); |
| |
| //band_position addition |
| band_table0_8x16b = _mm_add_epi16(band_table0_8x16b, band_pos_16x8b); |
| band_table1_8x16b = _mm_add_epi16(band_table1_8x16b, band_pos_16x8b); |
| band_table2_8x16b = _mm_add_epi16(band_table2_8x16b, band_pos_16x8b); |
| band_table3_8x16b = _mm_add_epi16(band_table3_8x16b, band_pos_16x8b); |
| //sao_offset duplication |
| tmp_set_128i_1 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_1); |
| tmp_set_128i_2 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_3); |
| tmp_set_128i_4 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_4); |
| //settng for comparision |
| cmp_mask = _mm_set1_epi16(16); |
| cmp_store = _mm_set1_epi16(0x00ff); |
| |
| //sao_offset addition |
| band_table0_8x16b = _mm_add_epi16(band_table0_8x16b, tmp_set_128i_1); |
| band_table1_8x16b = _mm_add_epi16(band_table1_8x16b, tmp_set_128i_2); |
| band_table2_8x16b = _mm_add_epi16(band_table2_8x16b, tmp_set_128i_3); |
| band_table3_8x16b = _mm_add_epi16(band_table3_8x16b, tmp_set_128i_4); |
| //masking upper 8bit values of each 16 bit band table value |
| band_table0_8x16b = _mm_and_si128(band_table0_8x16b, cmp_store); |
| band_table1_8x16b = _mm_and_si128(band_table1_8x16b, cmp_store); |
| band_table2_8x16b = _mm_and_si128(band_table2_8x16b, cmp_store); |
| band_table3_8x16b = _mm_and_si128(band_table3_8x16b, cmp_store); |
| |
| switch(sao_band_pos) |
| { |
| case 0: |
| tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table0_8x16b); |
| band_table0_8x16b = _mm_and_si128(band_table0_8x16b, tmp_set_128i_2); |
| break; |
| case 28: |
| tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table3_8x16b); |
| band_table3_8x16b = _mm_or_si128(band_table3_8x16b, tmp_set_128i_2); |
| break; |
| case 29: |
| tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table2_8x16b); |
| band_table2_8x16b = _mm_or_si128(band_table2_8x16b, tmp_set_128i_2); |
| tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table3_8x16b); |
| band_table3_8x16b = _mm_and_si128(band_table3_8x16b, tmp_set_128i_2); |
| break; |
| case 30: |
| tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table1_8x16b); |
| band_table1_8x16b = _mm_or_si128(band_table1_8x16b, tmp_set_128i_2); |
| tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table2_8x16b); |
| band_table2_8x16b = _mm_and_si128(band_table2_8x16b, tmp_set_128i_2); |
| break; |
| case 31: |
| tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table0_8x16b); |
| band_table0_8x16b = _mm_or_si128(band_table0_8x16b, tmp_set_128i_2); |
| tmp_set_128i_2 = _mm_cmpgt_epi16(cmp_mask, band_table1_8x16b); |
| band_table1_8x16b = _mm_and_si128(band_table1_8x16b, tmp_set_128i_2); |
| break; |
| default: |
| break; |
| } |
| //sao_offset is reused for zero cmp mask. |
| sao_offset = _mm_setzero_si128(); |
| tmp_set_128i_1 = _mm_set1_epi8(1); |
| //tmp_set_128i_2 = _mm_set_epi8 (128,7,128,6,128,5,128,4,128,3,128,2,128,1,128,0); |
| cmp_mask = _mm_packus_epi16(cmp_mask, cmp_mask); //cmp_msk=dup16(16); |
| |
| //masking upper 8bit values of each 16 bit band table value |
| band_table0_8x16b = _mm_and_si128(band_table0_8x16b, cmp_store); |
| band_table1_8x16b = _mm_and_si128(band_table1_8x16b, cmp_store); |
| band_table2_8x16b = _mm_and_si128(band_table2_8x16b, cmp_store); |
| band_table3_8x16b = _mm_and_si128(band_table3_8x16b, cmp_store); |
| |
| //band table 8x16 four registers are packed into 16x8 two registers: band_table0_8x16b and band_table2_8x16b |
| band_table0_8x16b = _mm_packus_epi16(band_table0_8x16b, band_table1_8x16b); |
| band_table2_8x16b = _mm_packus_epi16(band_table2_8x16b, band_table3_8x16b); |
| |
| band_table3_8x16b = _mm_slli_epi16(cmp_mask, 1); // to compare if value is greater than 31 |
| band_pos_16x8b = _mm_packus_epi16(band_pos_16x8b, band_pos_16x8b); //band_pos is now 8 bit aligned |
| band_table3_8x16b = _mm_sub_epi8(band_table3_8x16b, tmp_set_128i_1); // to compare if value is greater than 31 |
| |
| cmp_mask = _mm_sub_epi8(cmp_mask, tmp_set_128i_1); |
| // band_pos_16x8b = _mm_or_si128(band_pos_16x8b,cmp_store); |
| |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| for(row = ht; row > 0; row -= 2) |
| { |
| |
| |
| //row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp0_8x16b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| // row = 1 |
| src_temp2_8x16b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| |
| |
| //saturated substract 8 bit |
| tmp_set_128i_1 = _mm_sub_epi8(src_temp0_8x16b, band_pos_16x8b); |
| tmp_set_128i_3 = _mm_sub_epi8(src_temp2_8x16b, band_pos_16x8b); |
| //if the values less than 0 put ff |
| tmp_set_128i_2 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_1); |
| tmp_set_128i_4 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_3); |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| //if the values gret=ater than 31 put ff |
| tmp_set_128i_2 = _mm_cmpgt_epi8(tmp_set_128i_1, band_table3_8x16b); |
| tmp_set_128i_4 = _mm_cmpgt_epi8(tmp_set_128i_3, band_table3_8x16b); |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| |
| |
| //row 0 and row1 |
| //if the values >16 then put ff ,cmp_mask = dup16(15) |
| cmp_store = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_mask); |
| //values 16 to 31 for row 0 & 1 but values <16 ==0 |
| tmp_set_128i_2 = _mm_and_si128(tmp_set_128i_1, cmp_store); |
| // values 0 to 15 for row 0 & 1 |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, cmp_store); |
| //values 16 to 31 for row 0 & 1 but values <16 masked to ff |
| cmp_store = _mm_cmpeq_epi8(tmp_set_128i_2, sao_offset); |
| tmp_set_128i_2 = _mm_or_si128(tmp_set_128i_2, cmp_store); |
| //row 2 and row 3 |
| //if the values >16 then put ff ,cmp_mask = dup16(15) |
| cmp_store = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_mask); |
| //values 16 to 31 for row 2 & 3 but values <16 ==0 |
| tmp_set_128i_4 = _mm_and_si128(tmp_set_128i_3, cmp_store); |
| // values 0 to 15 for row 2 & 3 |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, cmp_store); |
| //values 16 to 31 for row 2 & 3 but values <16 masked to ff |
| cmp_store = _mm_cmpeq_epi8(tmp_set_128i_4, sao_offset); |
| tmp_set_128i_4 = _mm_or_si128(tmp_set_128i_4, cmp_store); |
| |
| //row 0 and row 1 |
| //to preserve pixel values in which no offset needs to be added. |
| cmp_store = _mm_cmpeq_epi8(tmp_set_128i_1, tmp_set_128i_2); |
| src_temp0_8x16b = _mm_and_si128(src_temp0_8x16b, cmp_store); |
| |
| //row 2 and row 3 |
| //to preserve pixel values in which no offset needs to be added. |
| cmp_store = _mm_cmpeq_epi8(tmp_set_128i_3, tmp_set_128i_4); |
| src_temp2_8x16b = _mm_and_si128(src_temp2_8x16b, cmp_store); |
| |
| //indexing 0 - 15 bandtable indexes |
| tmp_set_128i_1 = _mm_shuffle_epi8(band_table0_8x16b, tmp_set_128i_1); |
| tmp_set_128i_3 = _mm_shuffle_epi8(band_table0_8x16b, tmp_set_128i_3); |
| tmp_set_128i_2 = _mm_shuffle_epi8(band_table2_8x16b, tmp_set_128i_2); |
| tmp_set_128i_4 = _mm_shuffle_epi8(band_table2_8x16b, tmp_set_128i_4); |
| // combining all offsets results |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| // combing results woth the pixel values |
| src_temp0_8x16b = _mm_or_si128(src_temp0_8x16b, tmp_set_128i_1); |
| src_temp2_8x16b = _mm_or_si128(src_temp2_8x16b, tmp_set_128i_3); |
| |
| |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_8x16b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp2_8x16b); |
| |
| pu1_src_cpy += (src_strd << 1); |
| } |
| pu1_src += 16; |
| } |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| {pu1_src_cpy = pu1_src; |
| for(row = ht; row > 0; row -= 4) |
| { |
| |
| |
| //row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp0_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy)); |
| // row = 1 |
| src_temp1_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_temp2_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| // row = 3 |
| src_temp3_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| //row0 and row1 packed and row2 and row3 packed |
| |
| src_temp0_8x16b = _mm_unpacklo_epi64(src_temp0_8x16b, src_temp1_8x16b); |
| src_temp2_8x16b = _mm_unpacklo_epi64(src_temp2_8x16b, src_temp3_8x16b); |
| |
| //saturated substract 8 bit |
| tmp_set_128i_1 = _mm_sub_epi8(src_temp0_8x16b, band_pos_16x8b); |
| tmp_set_128i_3 = _mm_sub_epi8(src_temp2_8x16b, band_pos_16x8b); |
| //if the values less than 0 put ff |
| tmp_set_128i_2 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_1); |
| tmp_set_128i_4 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_3); |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| //if the values gret=ater than 31 put ff |
| tmp_set_128i_2 = _mm_cmpgt_epi8(tmp_set_128i_1, band_table3_8x16b); |
| tmp_set_128i_4 = _mm_cmpgt_epi8(tmp_set_128i_3, band_table3_8x16b); |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| |
| |
| |
| //row 0 and row1 |
| //if the values >16 then put ff ,cmp_mask = dup16(15) |
| cmp_store = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_mask); |
| //values 16 to 31 for row 0 & 1 but values <16 ==0 |
| tmp_set_128i_2 = _mm_and_si128(tmp_set_128i_1, cmp_store); |
| // values 0 to 15 for row 0 & 1 |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, cmp_store); |
| //values 16 to 31 for row 0 & 1 but values <16 masked to ff |
| cmp_store = _mm_cmpeq_epi8(tmp_set_128i_2, sao_offset); |
| tmp_set_128i_2 = _mm_or_si128(tmp_set_128i_2, cmp_store); |
| //row 2 and row 3 |
| //if the values >16 then put ff ,cmp_mask = dup16(15) |
| cmp_store = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_mask); |
| //values 16 to 31 for row 2 & 3 but values <16 ==0 |
| tmp_set_128i_4 = _mm_and_si128(tmp_set_128i_3, cmp_store); |
| // values 0 to 15 for row 2 & 3 |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, cmp_store); |
| //values 16 to 31 for row 2 & 3 but values <16 masked to ff |
| cmp_store = _mm_cmpeq_epi8(tmp_set_128i_4, sao_offset); |
| tmp_set_128i_4 = _mm_or_si128(tmp_set_128i_4, cmp_store); |
| |
| //row 0 and row 1 |
| //to preserve pixel values in which no offset needs to be added. |
| cmp_store = _mm_cmpeq_epi8(tmp_set_128i_1, tmp_set_128i_2); |
| src_temp0_8x16b = _mm_and_si128(src_temp0_8x16b, cmp_store); |
| |
| //row 2 and row 3 |
| //to preserve pixel values in which no offset needs to be added. |
| cmp_store = _mm_cmpeq_epi8(tmp_set_128i_3, tmp_set_128i_4); |
| src_temp2_8x16b = _mm_and_si128(src_temp2_8x16b, cmp_store); |
| |
| //indexing 0 - 15 bandtable indexes |
| tmp_set_128i_1 = _mm_shuffle_epi8(band_table0_8x16b, tmp_set_128i_1); |
| tmp_set_128i_3 = _mm_shuffle_epi8(band_table0_8x16b, tmp_set_128i_3); |
| tmp_set_128i_2 = _mm_shuffle_epi8(band_table2_8x16b, tmp_set_128i_2); |
| tmp_set_128i_4 = _mm_shuffle_epi8(band_table2_8x16b, tmp_set_128i_4); |
| // combining all offsets results |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| // combing results woth the pixel values |
| src_temp0_8x16b = _mm_or_si128(src_temp0_8x16b, tmp_set_128i_1); |
| src_temp2_8x16b = _mm_or_si128(src_temp2_8x16b, tmp_set_128i_3); |
| |
| //Getting row1 separately |
| src_temp1_8x16b = _mm_srli_si128(src_temp0_8x16b, 8); |
| //Getting row3 separately |
| src_temp3_8x16b = _mm_srli_si128(src_temp2_8x16b, 8); |
| |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_8x16b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), src_temp1_8x16b); |
| // row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_temp2_8x16b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), src_temp3_8x16b); |
| |
| pu1_src_cpy += (src_strd << 2); |
| |
| } |
| pu1_src += 8; |
| } |
| |
| |
| } |
| |
| void ihevc_sao_band_offset_chroma_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| WORD32 sao_band_pos_u, |
| WORD32 sao_band_pos_v, |
| WORD8 *pi1_sao_offset_u, |
| WORD8 *pi1_sao_offset_v, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| WORD8 offset = 0; |
| |
| |
| __m128i src_temp0_8x16b, src_temp1_8x16b, src_temp2_8x16b, src_temp3_8x16b; |
| __m128i cmp_msk2; |
| __m128i band_table0_16x8b, band_table1_16x8b, band_table2_16x8b, band_table3_16x8b; |
| __m128i tmp_set_128i_1, tmp_set_128i_2, tmp_set_128i_3, tmp_set_128i_4; |
| __m128i band_pos_u_16x8b, band_pos_v_16x8b; |
| __m128i sao_offset; |
| __m128i cmp_mask; |
| |
| |
| /* Updating left and top and top-left */ |
| for(row = 0; row < ht; row++) |
| { |
| pu1_src_left[2 * row] = pu1_src[row * src_strd + (wd - 2)]; |
| pu1_src_left[2 * row + 1] = pu1_src[row * src_strd + (wd - 1)]; |
| } |
| pu1_src_top_left[0] = pu1_src_top[wd - 2]; |
| pu1_src_top_left[1] = pu1_src_top[wd - 1]; |
| for(col = 0; col < wd; col += 8) |
| { |
| tmp_set_128i_1 = _mm_loadl_epi64((__m128i *)(pu1_src + (ht - 1) * src_strd + offset)); |
| _mm_storel_epi64((__m128i *)(pu1_src_top + offset), tmp_set_128i_1); |
| offset += 8; |
| } |
| |
| { // band _table creation |
| __m128i temp0_8x16b, temp1_8x16b, temp2_8x16b, temp3_8x16b; |
| // Band table for U component : band_table0_16x8b and band_table2_16x8b |
| //replicating sao_band_pos as 8 bit value 16 times |
| band_pos_u_16x8b = _mm_set1_epi16((WORD16)(sao_band_pos_u << 3)); |
| //value set for sao_offset extraction |
| tmp_set_128i_1 = _mm_set_epi8(128, 1, 128, 1, 128, 1, 128, 1, 128, 1, 128, 1, 128, 1, 128, 1); |
| tmp_set_128i_2 = _mm_set_epi8(128, 2, 128, 2, 128, 2, 128, 2, 128, 2, 128, 2, 128, 2, 128, 2); |
| tmp_set_128i_3 = _mm_set_epi8(128, 3, 128, 3, 128, 3, 128, 3, 128, 3, 128, 3, 128, 3, 128, 3); |
| tmp_set_128i_4 = _mm_set_epi8(128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4, 128, 4); |
| |
| //loaded sao offset values |
| sao_offset = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u); |
| |
| //loading 16bit 32values of gu2_table_band_idx consecutively in 4 registers |
| band_table0_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx)); |
| band_table1_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 8)); |
| band_table2_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 16)); |
| band_table3_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 24)); |
| |
| //band_position addition |
| band_table0_16x8b = _mm_add_epi16(band_table0_16x8b, band_pos_u_16x8b); |
| band_table1_16x8b = _mm_add_epi16(band_table1_16x8b, band_pos_u_16x8b); |
| band_table2_16x8b = _mm_add_epi16(band_table2_16x8b, band_pos_u_16x8b); |
| band_table3_16x8b = _mm_add_epi16(band_table3_16x8b, band_pos_u_16x8b); |
| //sao_offset duplication |
| temp0_8x16b = _mm_shuffle_epi8(sao_offset, tmp_set_128i_1); |
| temp1_8x16b = _mm_shuffle_epi8(sao_offset, tmp_set_128i_2); |
| temp2_8x16b = _mm_shuffle_epi8(sao_offset, tmp_set_128i_3); |
| temp3_8x16b = _mm_shuffle_epi8(sao_offset, tmp_set_128i_4); |
| |
| //sao_offset addition |
| band_table0_16x8b = _mm_add_epi16(band_table0_16x8b, temp0_8x16b); |
| band_table1_16x8b = _mm_add_epi16(band_table1_16x8b, temp1_8x16b); |
| band_table2_16x8b = _mm_add_epi16(band_table2_16x8b, temp2_8x16b); |
| band_table3_16x8b = _mm_add_epi16(band_table3_16x8b, temp3_8x16b); |
| //reuse for clipping |
| temp1_8x16b = _mm_set1_epi16(0x00ff); |
| //settng for comparision |
| cmp_mask = _mm_set1_epi16(16); |
| |
| //masking upper 8bit values of each 16 bit band table value |
| band_table0_16x8b = _mm_and_si128(band_table0_16x8b, temp1_8x16b); |
| band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp1_8x16b); |
| band_table2_16x8b = _mm_and_si128(band_table2_16x8b, temp1_8x16b); |
| band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp1_8x16b); |
| |
| //temp1_8x16b reuse for compare storage |
| switch(sao_band_pos_u) |
| { |
| case 0: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table0_16x8b); |
| band_table0_16x8b = _mm_and_si128(band_table0_16x8b, temp3_8x16b); |
| break; |
| case 28: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table3_16x8b); |
| band_table3_16x8b = _mm_or_si128(band_table3_16x8b, temp3_8x16b); |
| break; |
| case 29: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table2_16x8b); |
| band_table2_16x8b = _mm_or_si128(band_table2_16x8b, temp3_8x16b); |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table3_16x8b); |
| band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp3_8x16b); |
| break; |
| case 30: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table1_16x8b); |
| band_table1_16x8b = _mm_or_si128(band_table1_16x8b, temp3_8x16b); |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table2_16x8b); |
| band_table2_16x8b = _mm_and_si128(band_table2_16x8b, temp3_8x16b); |
| break; |
| case 31: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table0_16x8b); |
| band_table0_16x8b = _mm_or_si128(band_table0_16x8b, temp3_8x16b); |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table1_16x8b); |
| band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp3_8x16b); |
| break; |
| default: |
| break; |
| } |
| //masking upper 8bit values of each 16 bit band table value |
| band_table0_16x8b = _mm_and_si128(band_table0_16x8b, temp1_8x16b); |
| band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp1_8x16b); |
| band_table2_16x8b = _mm_and_si128(band_table2_16x8b, temp1_8x16b); |
| band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp1_8x16b); |
| //band table 8x16 four registers are packed into 16x8 two registers: band_table0_8x16b and band_table2_8x16b |
| band_table0_16x8b = _mm_packus_epi16(band_table0_16x8b, band_table1_16x8b); |
| band_table2_16x8b = _mm_packus_epi16(band_table2_16x8b, band_table3_16x8b); |
| // Band table for U component over |
| |
| // Band table for V component : band_table1_16x8b and band_table3_16x8b |
| // replicating sao_band_pos as 8 bit value 16 times |
| band_pos_v_16x8b = _mm_set1_epi16((WORD16)(sao_band_pos_v << 3)); |
| |
| //loaded sao offset values |
| sao_offset = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v); |
| |
| //loading 16bit 32values of gu2_table_band_idx consecutively in 4 registers |
| temp0_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx)); |
| band_table1_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 8)); |
| temp2_8x16b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 16)); |
| band_table3_16x8b = _mm_load_si128((__m128i *)(gu2_table_band_idx + 24)); |
| |
| //band_position addition |
| temp0_8x16b = _mm_add_epi16(temp0_8x16b, band_pos_v_16x8b); |
| band_table1_16x8b = _mm_add_epi16(band_table1_16x8b, band_pos_v_16x8b); |
| temp2_8x16b = _mm_add_epi16(temp2_8x16b, band_pos_v_16x8b); |
| band_table3_16x8b = _mm_add_epi16(band_table3_16x8b, band_pos_v_16x8b); |
| //sao_offset duplication |
| tmp_set_128i_1 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_1); |
| tmp_set_128i_2 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_3); |
| tmp_set_128i_4 = _mm_shuffle_epi8(sao_offset, tmp_set_128i_4); |
| |
| //sao_offset addition |
| temp0_8x16b = _mm_add_epi16(temp0_8x16b, tmp_set_128i_1); |
| band_table1_16x8b = _mm_add_epi16(band_table1_16x8b, tmp_set_128i_2); |
| temp2_8x16b = _mm_add_epi16(temp2_8x16b, tmp_set_128i_3); |
| band_table3_16x8b = _mm_add_epi16(band_table3_16x8b, tmp_set_128i_4); |
| |
| //masking upper 8bit values of 16 bit band table value |
| temp0_8x16b = _mm_and_si128(temp0_8x16b, temp1_8x16b); |
| band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp1_8x16b); |
| temp2_8x16b = _mm_and_si128(temp2_8x16b, temp1_8x16b); |
| band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp1_8x16b); |
| //temp1_8x16b reuse for compare storage |
| |
| switch(sao_band_pos_v) |
| { |
| case 0: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, temp0_8x16b); |
| temp0_8x16b = _mm_and_si128(temp0_8x16b, temp3_8x16b); |
| break; |
| case 28: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table3_16x8b); |
| band_table3_16x8b = _mm_or_si128(band_table3_16x8b, temp3_8x16b); |
| break; |
| case 29: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, temp2_8x16b); |
| temp2_8x16b = _mm_or_si128(temp2_8x16b, temp3_8x16b); |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table3_16x8b); |
| band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp3_8x16b); |
| break; |
| case 30: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table1_16x8b); |
| band_table1_16x8b = _mm_or_si128(band_table1_16x8b, temp3_8x16b); |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, temp2_8x16b); |
| temp2_8x16b = _mm_and_si128(temp2_8x16b, temp3_8x16b); |
| break; |
| case 31: |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, temp0_8x16b); |
| temp0_8x16b = _mm_or_si128(temp0_8x16b, temp3_8x16b); |
| temp3_8x16b = _mm_cmpgt_epi16(cmp_mask, band_table1_16x8b); |
| band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp3_8x16b); |
| break; |
| default: |
| break; |
| } |
| //masking upper 8bit values of each 16 bit band table value |
| temp0_8x16b = _mm_and_si128(temp0_8x16b, temp1_8x16b); |
| band_table1_16x8b = _mm_and_si128(band_table1_16x8b, temp1_8x16b); |
| temp2_8x16b = _mm_and_si128(temp2_8x16b, temp1_8x16b); |
| band_table3_16x8b = _mm_and_si128(band_table3_16x8b, temp1_8x16b); |
| //band table 8x16 four registers are packed into 16x8 two registers: band_table0_8x16b and band_table2_8x16b |
| band_table1_16x8b = _mm_packus_epi16(temp0_8x16b, band_table1_16x8b); |
| band_table3_16x8b = _mm_packus_epi16(temp2_8x16b, band_table3_16x8b); |
| //band table for u and v created |
| } |
| { |
| UWORD8 *pu1_src_cpy; |
| WORD32 wd_rem; |
| |
| |
| //sao_offset is reused for zero cmp mask. |
| sao_offset = _mm_setzero_si128(); |
| tmp_set_128i_1 = _mm_set1_epi8(1); |
| //tmp_set_128i_2 = _mm_set_epi8 (128,7,128,6,128,5,128,4,128,3,128,2,128,1,128,0); |
| cmp_mask = _mm_packus_epi16(cmp_mask, cmp_mask); //cmp_msk=dup16(16); |
| //to avoid ffff to be saturated to 0 instead it should be to ff |
| |
| cmp_msk2 = _mm_slli_epi16(cmp_mask, 1); // to compare if value is greater than 31 |
| band_pos_u_16x8b = _mm_packus_epi16(band_pos_u_16x8b, band_pos_u_16x8b); //band_pos_u is now 8 bit aligned |
| band_pos_v_16x8b = _mm_packus_epi16(band_pos_v_16x8b, band_pos_v_16x8b); //band_pos_v is now 8 bit aligned |
| cmp_msk2 = _mm_sub_epi8(cmp_msk2, tmp_set_128i_1); // to compare if value is greater than 31 |
| |
| cmp_mask = _mm_sub_epi8(cmp_mask, tmp_set_128i_1); |
| |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| for(row = ht; row > 0; row -= 2) |
| { |
| //row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp0_8x16b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| // row = 1 |
| src_temp3_8x16b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| |
| //odd values |
| src_temp1_8x16b = _mm_srli_epi16(src_temp0_8x16b, 8); |
| src_temp2_8x16b = _mm_srli_epi16(src_temp3_8x16b, 8); |
| //even values |
| src_temp0_8x16b = _mm_slli_epi16(src_temp0_8x16b, 8); |
| src_temp3_8x16b = _mm_slli_epi16(src_temp3_8x16b, 8); |
| src_temp0_8x16b = _mm_srli_epi16(src_temp0_8x16b, 8); |
| src_temp3_8x16b = _mm_srli_epi16(src_temp3_8x16b, 8); |
| //combining odd values |
| src_temp2_8x16b = _mm_packus_epi16(src_temp1_8x16b, src_temp2_8x16b); |
| //combining even values |
| src_temp0_8x16b = _mm_packus_epi16(src_temp0_8x16b, src_temp3_8x16b); |
| |
| //saturated substract 8 bit |
| tmp_set_128i_1 = _mm_sub_epi8(src_temp0_8x16b, band_pos_u_16x8b); |
| tmp_set_128i_3 = _mm_sub_epi8(src_temp2_8x16b, band_pos_v_16x8b); |
| //if the values less than 0 put ff |
| tmp_set_128i_2 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_1); |
| tmp_set_128i_4 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_3); |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| //if the values greater than 31 put ff |
| tmp_set_128i_2 = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_msk2); |
| tmp_set_128i_4 = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_msk2); |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| // registers reused to increase performance |
| //if the values >16 then put ff ,cmp_mask = dup16(15) row 0 and row1 |
| src_temp1_8x16b = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_mask); |
| //if the values >16 then put ff ,cmp_mask = dup16(15) row 2 and row 3 |
| src_temp3_8x16b = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_mask); |
| |
| //values 16 to 31 for row 0 & 1 but values <16 ==0 |
| tmp_set_128i_2 = _mm_and_si128(tmp_set_128i_1, src_temp1_8x16b); |
| // values 0 to 15 for row 0 & 1 |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, src_temp1_8x16b); |
| //values 16 to 31 for row 2 & 3 but values <16 ==0 |
| tmp_set_128i_4 = _mm_and_si128(tmp_set_128i_3, src_temp3_8x16b); |
| // values 0 to 15 for row 2 & 3 |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, src_temp3_8x16b); |
| |
| //values 16 to 31 for row 0 & 1 but values <16 masked to ff row 0 and row1 |
| src_temp1_8x16b = _mm_cmpeq_epi8(tmp_set_128i_2, sao_offset); |
| //values 16 to 31 for row 0 & 1 but values <16 masked to ff row 2 and row 3 |
| src_temp3_8x16b = _mm_cmpeq_epi8(tmp_set_128i_4, sao_offset); |
| tmp_set_128i_2 = _mm_or_si128(tmp_set_128i_2, src_temp1_8x16b); |
| tmp_set_128i_4 = _mm_or_si128(tmp_set_128i_4, src_temp3_8x16b); |
| |
| |
| //to choose which pixel values to preserve in row 0 and row 1 |
| src_temp1_8x16b = _mm_cmpeq_epi8(tmp_set_128i_1, tmp_set_128i_2); |
| //to choose which pixel values to preserve in row 2 and row 3 |
| src_temp3_8x16b = _mm_cmpeq_epi8(tmp_set_128i_3, tmp_set_128i_4); |
| //values of all rows to which no offset needs to be added preserved. |
| src_temp0_8x16b = _mm_and_si128(src_temp0_8x16b, src_temp1_8x16b); |
| src_temp2_8x16b = _mm_and_si128(src_temp2_8x16b, src_temp3_8x16b); |
| |
| //indexing 0 - 15 bandtable indexes |
| tmp_set_128i_1 = _mm_shuffle_epi8(band_table0_16x8b, tmp_set_128i_1); //U low |
| tmp_set_128i_3 = _mm_shuffle_epi8(band_table1_16x8b, tmp_set_128i_3); //V low |
| //indexing 16 -31 bandtable indexes |
| tmp_set_128i_2 = _mm_shuffle_epi8(band_table2_16x8b, tmp_set_128i_2); //U high |
| tmp_set_128i_4 = _mm_shuffle_epi8(band_table3_16x8b, tmp_set_128i_4); //V high |
| // combining all offsets results |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); //U |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); //V |
| // combing results with the pixel values |
| src_temp0_8x16b = _mm_or_si128(src_temp0_8x16b, tmp_set_128i_1); |
| src_temp2_8x16b = _mm_or_si128(src_temp2_8x16b, tmp_set_128i_3); |
| //reorganising even and odd values |
| src_temp1_8x16b = _mm_unpacklo_epi8(src_temp0_8x16b, src_temp2_8x16b); |
| src_temp3_8x16b = _mm_unpackhi_epi8(src_temp0_8x16b, src_temp2_8x16b); |
| |
| |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp1_8x16b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp3_8x16b); |
| |
| |
| pu1_src_cpy += (src_strd << 1); |
| |
| } |
| pu1_src += 16; |
| } |
| |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| pu1_src_cpy = pu1_src; |
| for(row = ht; row > 0; row -= 4) |
| { |
| //row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp0_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy)); |
| // row = 1 |
| src_temp1_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_temp2_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| // row = 3 |
| src_temp3_8x16b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| //row0 and row1 packed and row2 and row3 packed |
| |
| src_temp0_8x16b = _mm_unpacklo_epi64(src_temp0_8x16b, src_temp1_8x16b); |
| src_temp3_8x16b = _mm_unpacklo_epi64(src_temp2_8x16b, src_temp3_8x16b); |
| //odd values |
| src_temp1_8x16b = _mm_srli_epi16(src_temp0_8x16b, 8); |
| src_temp2_8x16b = _mm_srli_epi16(src_temp3_8x16b, 8); |
| //even values |
| src_temp0_8x16b = _mm_slli_epi16(src_temp0_8x16b, 8); |
| src_temp3_8x16b = _mm_slli_epi16(src_temp3_8x16b, 8); |
| src_temp0_8x16b = _mm_srli_epi16(src_temp0_8x16b, 8); |
| src_temp3_8x16b = _mm_srli_epi16(src_temp3_8x16b, 8); |
| //combining odd values |
| src_temp2_8x16b = _mm_packus_epi16(src_temp1_8x16b, src_temp2_8x16b); |
| //combining even values |
| src_temp0_8x16b = _mm_packus_epi16(src_temp0_8x16b, src_temp3_8x16b); |
| |
| //saturated substract 8 bit |
| tmp_set_128i_1 = _mm_sub_epi8(src_temp0_8x16b, band_pos_u_16x8b); |
| tmp_set_128i_3 = _mm_sub_epi8(src_temp2_8x16b, band_pos_v_16x8b); |
| //if the values less than 0 put ff |
| tmp_set_128i_2 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_1); |
| tmp_set_128i_4 = _mm_cmpgt_epi8(sao_offset, tmp_set_128i_3); |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| //if the values greater than 31 put ff |
| tmp_set_128i_2 = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_msk2); |
| tmp_set_128i_4 = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_msk2); |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); |
| // registers reused to increase performance |
| //if the values >16 then put ff ,cmp_mask = dup16(15) row 0 and row1 |
| src_temp1_8x16b = _mm_cmpgt_epi8(tmp_set_128i_1, cmp_mask); |
| //if the values >16 then put ff ,cmp_mask = dup16(15) row 2 and row 3 |
| src_temp3_8x16b = _mm_cmpgt_epi8(tmp_set_128i_3, cmp_mask); |
| |
| //values 16 to 31 for row 0 & 1 but values <16 ==0 |
| tmp_set_128i_2 = _mm_and_si128(tmp_set_128i_1, src_temp1_8x16b); |
| // values 0 to 15 for row 0 & 1 |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, src_temp1_8x16b); |
| //values 16 to 31 for row 2 & 3 but values <16 ==0 |
| tmp_set_128i_4 = _mm_and_si128(tmp_set_128i_3, src_temp3_8x16b); |
| // values 0 to 15 for row 2 & 3 |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, src_temp3_8x16b); |
| |
| //values 16 to 31 for row 0 & 1 but values <16 masked to ff row 0 and row1 |
| src_temp1_8x16b = _mm_cmpeq_epi8(tmp_set_128i_2, sao_offset); |
| //values 16 to 31 for row 0 & 1 but values <16 masked to ff row 2 and row 3 |
| src_temp3_8x16b = _mm_cmpeq_epi8(tmp_set_128i_4, sao_offset); |
| tmp_set_128i_2 = _mm_or_si128(tmp_set_128i_2, src_temp1_8x16b); |
| tmp_set_128i_4 = _mm_or_si128(tmp_set_128i_4, src_temp3_8x16b); |
| |
| |
| //to choose which pixel values to preserve in row 0 and row 1 |
| src_temp1_8x16b = _mm_cmpeq_epi8(tmp_set_128i_1, tmp_set_128i_2); |
| //to choose which pixel values to preserve in row 2 and row 3 |
| src_temp3_8x16b = _mm_cmpeq_epi8(tmp_set_128i_3, tmp_set_128i_4); |
| //values of all rows to which no offset needs to be added preserved. |
| src_temp0_8x16b = _mm_and_si128(src_temp0_8x16b, src_temp1_8x16b); |
| src_temp2_8x16b = _mm_and_si128(src_temp2_8x16b, src_temp3_8x16b); |
| |
| //indexing 0 - 15 bandtable indexes |
| tmp_set_128i_1 = _mm_shuffle_epi8(band_table0_16x8b, tmp_set_128i_1); //U low |
| tmp_set_128i_3 = _mm_shuffle_epi8(band_table1_16x8b, tmp_set_128i_3); //V low |
| //indexing 16 -31 bandtable indexes |
| tmp_set_128i_2 = _mm_shuffle_epi8(band_table2_16x8b, tmp_set_128i_2); //U high |
| tmp_set_128i_4 = _mm_shuffle_epi8(band_table3_16x8b, tmp_set_128i_4); //V high |
| // combining all offsets results |
| tmp_set_128i_1 = _mm_or_si128(tmp_set_128i_1, tmp_set_128i_2); //U |
| tmp_set_128i_3 = _mm_or_si128(tmp_set_128i_3, tmp_set_128i_4); //V |
| // combing results with the pixel values |
| src_temp0_8x16b = _mm_or_si128(src_temp0_8x16b, tmp_set_128i_1); |
| src_temp2_8x16b = _mm_or_si128(src_temp2_8x16b, tmp_set_128i_3); |
| //reorganising even and odd values |
| src_temp1_8x16b = _mm_unpacklo_epi8(src_temp0_8x16b, src_temp2_8x16b); |
| src_temp3_8x16b = _mm_unpackhi_epi8(src_temp0_8x16b, src_temp2_8x16b); |
| //Getting row1 separately |
| src_temp0_8x16b = _mm_srli_si128(src_temp1_8x16b, 8); |
| //Getting row3 separately |
| src_temp2_8x16b = _mm_srli_si128(src_temp3_8x16b, 8); |
| |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp1_8x16b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), src_temp0_8x16b); |
| // row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_temp3_8x16b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), src_temp2_8x16b); |
| |
| pu1_src_cpy += (src_strd << 2); |
| |
| } |
| pu1_src += 16; |
| } |
| |
| |
| } |
| } |
| |
| |
| |
| void ihevc_sao_edge_offset_class0_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| UWORD8 *pu1_src_top_right, |
| UWORD8 *pu1_src_bot_left, |
| UWORD8 *pu1_avail, |
| WORD8 *pi1_sao_offset, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_cpy, *pu1_src_left_cpy, *pu1_src_left_str, *pu1_left_tmp; |
| UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy; |
| UWORD8 au1_src_left_tmp[MAX_CTB_SIZE + 8]; |
| UWORD8 au1_src_left_tmp1[MAX_CTB_SIZE + 8]; |
| UWORD8 u1_avail0, u1_avail1; |
| WORD32 wd_rem; |
| WORD32 offset = 0; |
| __m128i src_temp0_16x8b, src_temp1_16x8b; |
| __m128i left0_16x8b, left1_16x8b; |
| __m128i cmp_gt0_16x8b, cmp_lt0_16x8b, cmp_gt1_16x8b, cmp_lt1_16x8b; |
| __m128i edge0_16x8b, edge1_16x8b; |
| __m128i au1_mask8x16b; |
| __m128i edge_idx_8x16b, sao_offset_8x16b; |
| __m128i const2_16x8b, const0_16x8b; |
| __m128i left_store_16x8b; |
| UNUSED(pu1_src_top_right); |
| UNUSED(pu1_src_bot_left); |
| |
| au1_mask8x16b = _mm_set1_epi8(0xff); |
| |
| /* Update top and top-left arrays */ |
| |
| *pu1_src_top_left = pu1_src_top[wd - 1]; |
| |
| for(col = wd; col >= 16; col -= 16) |
| { |
| const0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src + offset + (ht - 1) * src_strd)); |
| _mm_storeu_si128((__m128i *)(pu1_src_top + offset), const0_16x8b); |
| offset += 16; |
| } |
| |
| //setting availability mask to ff size MAX_CTB_SIZE |
| for(col = 0; col < MAX_CTB_SIZE; col += 16) |
| _mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b); |
| for(row = 0; row < ht; row++) |
| { |
| au1_src_left_tmp[row] = pu1_src_left[row]; |
| } |
| edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx); |
| sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset); |
| |
| //availability mask creation |
| u1_avail0 = pu1_avail[0]; |
| u1_avail1 = pu1_avail[1]; |
| au1_mask[0] = u1_avail0; |
| au1_mask[wd - 1] = u1_avail1; |
| |
| const2_16x8b = _mm_set1_epi8(2); |
| const0_16x8b = _mm_setzero_si128(); |
| pu1_src_left_cpy = au1_src_left_tmp; |
| pu1_src_left_str = au1_src_left_tmp1; |
| { |
| au1_mask_cpy = au1_mask; |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy); |
| //pu1_src_left_cpy =au1_src_left_tmp; |
| for(row = ht; row > 0; row -= 2) |
| { |
| |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy)); |
| //row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| // row = 1 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, left_store_16x8b, 2); |
| //row 1 left |
| left1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, left_store_16x8b, 15); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 15); |
| //row 0 left |
| left0_16x8b = _mm_alignr_epi8(src_temp0_16x8b, left_store_16x8b, 15); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| |
| |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, left0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(left0_16x8b, src_temp0_16x8b); |
| cmp_gt1_16x8b = _mm_subs_epu8(src_temp1_16x8b, left1_16x8b); |
| cmp_lt1_16x8b = _mm_subs_epu8(left1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| cmp_gt1_16x8b = _mm_cmpeq_epi8(cmp_gt1_16x8b, const0_16x8b); |
| cmp_lt1_16x8b = _mm_cmpeq_epi8(cmp_lt1_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| left0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| left1_16x8b = _mm_sub_epi8(cmp_gt1_16x8b, cmp_lt1_16x8b); |
| |
| //row = 0 right |
| edge0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 1)); |
| // row = 1 right |
| edge1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 1)); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, edge0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(edge0_16x8b, src_temp0_16x8b); |
| cmp_gt1_16x8b = _mm_subs_epu8(src_temp1_16x8b, edge1_16x8b); |
| cmp_lt1_16x8b = _mm_subs_epu8(edge1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| cmp_gt1_16x8b = _mm_cmpeq_epi8(cmp_gt1_16x8b, const0_16x8b); |
| cmp_lt1_16x8b = _mm_cmpeq_epi8(cmp_lt1_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| edge1_16x8b = _mm_sub_epi8(cmp_gt1_16x8b, cmp_lt1_16x8b); |
| |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, left0_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, left1_16x8b); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //cnvert to 16 bit then add and then saturated pack |
| left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, left0_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, left0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, left0_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, left0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b); |
| |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 2; |
| pu1_src_left_str += 2; |
| } |
| au1_mask_cpy += 16; |
| pu1_src += 16; |
| pu1_src_left_cpy -= ht; |
| pu1_src_left_str -= ht; |
| |
| pu1_left_tmp = pu1_src_left_cpy; |
| pu1_src_left_cpy = pu1_src_left_str; |
| pu1_src_left_str = pu1_left_tmp; |
| } |
| |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| |
| cmp_gt1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src + (ht - 1) * src_strd)); |
| _mm_storel_epi64((__m128i *)(pu1_src_top + offset), cmp_gt1_16x8b); |
| |
| au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); |
| pu1_src_cpy = pu1_src; |
| au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b); |
| //pu1_src_left_cpy =au1_src_left_tmp; |
| for(row = ht; row > 0; row -= 4) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy)); |
| //row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy)); |
| // row = 1 |
| cmp_gt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| // row = 3 |
| cmp_gt1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| |
| |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, left_store_16x8b, 4); |
| //row 3 left |
| edge0_16x8b = _mm_slli_si128(cmp_gt1_16x8b, 8); |
| cmp_lt1_16x8b = _mm_alignr_epi8(cmp_gt1_16x8b, left_store_16x8b, 15); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 15); |
| //row 2 left |
| edge0_16x8b = _mm_slli_si128(src_temp1_16x8b, 8); |
| left1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, left_store_16x8b, 15); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 15); |
| //row 1 left |
| edge0_16x8b = _mm_slli_si128(cmp_gt0_16x8b, 8); |
| cmp_lt0_16x8b = _mm_alignr_epi8(cmp_gt0_16x8b, left_store_16x8b, 15); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 15); |
| //row 0 left |
| edge0_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| left0_16x8b = _mm_alignr_epi8(src_temp0_16x8b, left_store_16x8b, 15); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 15); |
| |
| // packing rows together for 16 SIMD operations |
| src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, cmp_gt0_16x8b); |
| src_temp1_16x8b = _mm_unpacklo_epi64(src_temp1_16x8b, cmp_gt1_16x8b); |
| // packing rows together for 16 SIMD operations |
| left0_16x8b = _mm_unpacklo_epi64(left0_16x8b, cmp_lt0_16x8b); |
| left1_16x8b = _mm_unpacklo_epi64(left1_16x8b, cmp_lt1_16x8b); |
| |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, left0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(left0_16x8b, src_temp0_16x8b); |
| cmp_gt1_16x8b = _mm_subs_epu8(src_temp1_16x8b, left1_16x8b); |
| cmp_lt1_16x8b = _mm_subs_epu8(left1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| cmp_gt1_16x8b = _mm_cmpeq_epi8(cmp_gt1_16x8b, const0_16x8b); |
| cmp_lt1_16x8b = _mm_cmpeq_epi8(cmp_lt1_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| left0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| left1_16x8b = _mm_sub_epi8(cmp_gt1_16x8b, cmp_lt1_16x8b); |
| |
| //row = 0 right |
| edge0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 1)); |
| // row = 1 right |
| cmp_gt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd + 1)); |
| // row = 2 right |
| edge1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd + 1)); |
| // row = 3 right |
| cmp_gt1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd + 1)); |
| // packing rows together for 16 SIMD operations |
| edge0_16x8b = _mm_unpacklo_epi64(edge0_16x8b, cmp_gt0_16x8b); |
| edge1_16x8b = _mm_unpacklo_epi64(edge1_16x8b, cmp_gt1_16x8b); |
| |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, edge0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(edge0_16x8b, src_temp0_16x8b); |
| cmp_gt1_16x8b = _mm_subs_epu8(src_temp1_16x8b, edge1_16x8b); |
| cmp_lt1_16x8b = _mm_subs_epu8(edge1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| cmp_gt1_16x8b = _mm_cmpeq_epi8(cmp_gt1_16x8b, const0_16x8b); |
| cmp_lt1_16x8b = _mm_cmpeq_epi8(cmp_lt1_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| edge1_16x8b = _mm_sub_epi8(cmp_gt1_16x8b, cmp_lt1_16x8b); |
| |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, left0_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, left1_16x8b); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //shuffle to get sao offset |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //cnvert to 16 bit then add and then saturated pack |
| left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, left0_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, left0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, left0_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, left0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| //separting row 1 and row 3 |
| cmp_lt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| cmp_lt1_16x8b = _mm_srli_si128(src_temp1_16x8b, 8); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_lt0_16x8b); |
| // row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_temp1_16x8b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt1_16x8b); |
| |
| pu1_src_cpy += (src_strd << 2); |
| pu1_src_left_cpy += 4; |
| pu1_src_left_str += 4; |
| } |
| pu1_src += wd; |
| pu1_src_left_cpy -= ht; |
| pu1_src_left_str -= ht; |
| |
| pu1_left_tmp = pu1_src_left_cpy; |
| pu1_src_left_cpy = pu1_src_left_str; |
| pu1_src_left_str = pu1_left_tmp; |
| } |
| for(row = 0; row < ht; row++) |
| { |
| pu1_src_left[row] = pu1_src_left_cpy[row]; |
| } |
| } |
| } |
| |
| |
| void ihevc_sao_edge_offset_class0_chroma_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| UWORD8 *pu1_src_top_right, |
| UWORD8 *pu1_src_bot_left, |
| UWORD8 *pu1_avail, |
| WORD8 *pi1_sao_offset_u, |
| WORD8 *pi1_sao_offset_v, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_cpy, *pu1_src_left_cpy, *pu1_src_left_str, *pu1_left_tmp; |
| UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy; |
| UWORD8 au1_src_left_tmp[2 * (MAX_CTB_SIZE + 8)]; |
| UWORD8 au1_src_left_tmp1[2 * (MAX_CTB_SIZE + 8)]; |
| UWORD8 u1_avail0, u1_avail1; |
| WORD32 wd_rem; |
| WORD32 offset = 0; |
| |
| __m128i src_temp0_16x8b, src_temp1_16x8b; |
| __m128i left0_16x8b, left1_16x8b; |
| __m128i cmp_gt0_16x8b, cmp_lt0_16x8b; |
| __m128i edge0_16x8b, edge1_16x8b; |
| __m128i au1_mask8x16b; |
| __m128i edge_idx_8x16b, sao_offset_8x16b; |
| __m128i const2_16x8b, const0_16x8b; |
| __m128i left_store_16x8b; |
| __m128i chroma_offset_8x16b; |
| UNUSED(pu1_src_top_right); |
| UNUSED(pu1_src_bot_left); |
| |
| au1_mask8x16b = _mm_set1_epi8(0xff); |
| |
| /* Update top and top-left arrays */ |
| pu1_src_top_left[0] = pu1_src_top[wd - 2]; |
| pu1_src_top_left[1] = pu1_src_top[wd - 1];; |
| |
| for(col = wd; col >= 16; col -= 16) |
| { |
| const0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src + offset + (ht - 1) * src_strd)); |
| _mm_storeu_si128((__m128i *)(pu1_src_top + offset), const0_16x8b); |
| offset += 16; |
| } |
| for(row = 0; row < 2 * ht; row++) |
| { |
| au1_src_left_tmp[row] = pu1_src_left[row]; |
| } |
| //setting availability mask to ff size MAX_CTB_SIZE |
| for(col = 0; col < MAX_CTB_SIZE; col += 16) |
| _mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b); |
| |
| edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx); |
| sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u); |
| const0_16x8b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v); |
| chroma_offset_8x16b = _mm_set1_epi16(0x0800); |
| //availability mask creation |
| u1_avail0 = pu1_avail[0]; |
| u1_avail1 = pu1_avail[1]; |
| au1_mask[0] = u1_avail0; |
| au1_mask[1] = u1_avail0; |
| au1_mask[wd - 1] = u1_avail1; |
| au1_mask[wd - 2] = u1_avail1; |
| sao_offset_8x16b = _mm_unpacklo_epi64(sao_offset_8x16b, const0_16x8b); |
| const2_16x8b = _mm_set1_epi8(2); |
| const0_16x8b = _mm_setzero_si128(); |
| |
| { |
| pu1_src_left_cpy = au1_src_left_tmp; |
| pu1_src_left_str = au1_src_left_tmp1; |
| au1_mask_cpy = au1_mask; |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy); |
| |
| for(row = ht; row > 0; row -= 2) |
| { |
| |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy)); |
| //row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| // row = 1 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, left_store_16x8b, 4); |
| //row 1 left |
| left1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, left_store_16x8b, 14); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 14); |
| //row 0 left |
| left0_16x8b = _mm_alignr_epi8(src_temp0_16x8b, left_store_16x8b, 14); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| |
| |
| //separating +ve and and -ve values.row 0 left |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, left0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(left0_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| left0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //separating +ve and and -ve values.row 1 left |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, left1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(left1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| left1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| |
| //row = 0 right |
| edge0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2)); |
| // row = 1 right |
| edge1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 2)); |
| //separating +ve and and -ve values.row 0 right |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, edge0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(edge0_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //separating +ve and and -ve values.row 1 right |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, edge1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(edge1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, left0_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, left1_16x8b); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //cnvert to 16 bit then add and then saturated pack |
| left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, left0_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, left0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, left0_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, left0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b); |
| |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 4; |
| pu1_src_left_str += 4; |
| } |
| au1_mask_cpy += 16; |
| pu1_src += 16; |
| pu1_src_left_cpy -= 2 * ht; |
| pu1_src_left_str -= 2 * ht; |
| |
| pu1_left_tmp = pu1_src_left_cpy; |
| pu1_src_left_cpy = pu1_src_left_str; |
| pu1_src_left_str = pu1_left_tmp; |
| } |
| |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| |
| cmp_gt0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src + (ht - 1) * src_strd)); |
| _mm_storel_epi64((__m128i *)(pu1_src_top + offset), cmp_gt0_16x8b); |
| |
| au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); |
| pu1_src_cpy = pu1_src; |
| au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b); |
| |
| for(row = ht; row > 0; row -= 4) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy)); |
| //row = 0 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy)); |
| // row = 1 |
| cmp_gt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| // row = 3 |
| cmp_lt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| |
| |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, left_store_16x8b, 8); |
| //row 3 left |
| edge0_16x8b = _mm_slli_si128(cmp_lt0_16x8b, 8); |
| left0_16x8b = _mm_alignr_epi8(cmp_lt0_16x8b, left_store_16x8b, 14); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 14); |
| //row 2 left |
| edge0_16x8b = _mm_slli_si128(src_temp1_16x8b, 8); |
| left1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, left_store_16x8b, 14); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 14); |
| |
| |
| // packing rows together for 16 SIMD operations |
| src_temp1_16x8b = _mm_unpacklo_epi64(src_temp1_16x8b, cmp_lt0_16x8b); |
| left1_16x8b = _mm_unpacklo_epi64(left1_16x8b, left0_16x8b); |
| |
| //row 1 left |
| edge0_16x8b = _mm_slli_si128(cmp_gt0_16x8b, 8); |
| edge1_16x8b = _mm_alignr_epi8(cmp_gt0_16x8b, left_store_16x8b, 14); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 14); |
| //row 0 left |
| edge0_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| left0_16x8b = _mm_alignr_epi8(src_temp0_16x8b, left_store_16x8b, 14); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, edge0_16x8b, 14); |
| // packing rows together for 16 SIMD operations |
| src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, cmp_gt0_16x8b); |
| left0_16x8b = _mm_unpacklo_epi64(left0_16x8b, edge1_16x8b); |
| |
| //separating +ve and and -ve values.for row 2 and row 3 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, left1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(left1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| left1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| |
| |
| |
| |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, left0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(left0_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| left0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| |
| //row = 0 right |
| edge0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2)); |
| // row = 1 right |
| cmp_gt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd + 2)); |
| // row = 2 right |
| edge1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd + 2)); |
| // row = 3 right |
| cmp_lt0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd + 2)); |
| // packing rows together for 16 SIMD operations |
| edge0_16x8b = _mm_unpacklo_epi64(edge0_16x8b, cmp_gt0_16x8b); |
| edge1_16x8b = _mm_unpacklo_epi64(edge1_16x8b, cmp_lt0_16x8b); |
| |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, edge0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(edge0_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, edge1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(edge1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, left0_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, left1_16x8b); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //shuffle to get sao offset |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b); |
| |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //cnvert to 16 bit then add and then saturated pack |
| left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, left0_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, left0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| left0_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, left0_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, left0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| |
| //seaprting row 1 and row 3 |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| cmp_lt0_16x8b = _mm_srli_si128(src_temp1_16x8b, 8); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| // row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_temp1_16x8b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b); |
| |
| pu1_src_cpy += (src_strd << 2); |
| pu1_src_left_cpy += 8; |
| pu1_src_left_str += 8; |
| } |
| pu1_src += wd; |
| pu1_src_left_cpy -= 2 * ht; |
| pu1_src_left_str -= 2 * ht; |
| |
| pu1_left_tmp = pu1_src_left_cpy; |
| pu1_src_left_cpy = pu1_src_left_str; |
| pu1_src_left_str = pu1_left_tmp; |
| } |
| for(row = 0; row < 2 * ht; row++) |
| { |
| pu1_src_left[row] = pu1_src_left_cpy[row]; |
| } |
| } |
| |
| } |
| |
| |
| void ihevc_sao_edge_offset_class1_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| UWORD8 *pu1_src_top_right, |
| UWORD8 *pu1_src_bot_left, |
| UWORD8 *pu1_avail, |
| WORD8 *pi1_sao_offset, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_top_cpy; |
| UWORD8 *pu1_src_cpy; |
| WORD32 wd_rem; |
| |
| |
| __m128i src_top_16x8b, src_bottom_16x8b; |
| __m128i src_temp0_16x8b, src_temp1_16x8b; |
| __m128i signup0_16x8b, signdwn1_16x8b; |
| __m128i cmp_gt0_16x8b, cmp_lt0_16x8b; |
| __m128i edge0_16x8b, edge1_16x8b; |
| __m128i edge_idx_8x16b, sao_offset_8x16b; |
| __m128i const2_16x8b, const0_16x8b; |
| |
| UNUSED(pu1_src_top_right); |
| UNUSED(pu1_src_bot_left); |
| |
| |
| /* Updating left and top-left */ |
| for(row = 0; row < ht; row++) |
| { |
| pu1_src_left[row] = pu1_src[row * src_strd + (wd - 1)]; |
| } |
| *pu1_src_top_left = pu1_src_top[wd - 1]; |
| |
| |
| |
| pu1_src_top_cpy = pu1_src_top; |
| edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx); |
| sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset); |
| |
| /* Update height and source pointers based on the availability flags */ |
| if(0 == pu1_avail[2]) |
| { |
| pu1_src_top_cpy = pu1_src; |
| pu1_src += src_strd; |
| ht--; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| ht--; |
| } |
| |
| const2_16x8b = _mm_set1_epi8(2); |
| const0_16x8b = _mm_setzero_si128(); |
| |
| { |
| WORD32 ht_rem; |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| for(row = ht; row >= 2; row -= 2) |
| { |
| |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); |
| |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b); |
| |
| //for the next iteration signup0_16x8b = -signdwn1_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //copying the next top |
| src_top_16x8b = src_temp1_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b); |
| |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| } |
| ht_rem = ht & 0x1; |
| |
| if(ht_rem) |
| { |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| //current row -next row |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and botton and constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| //copying the next top |
| src_top_16x8b = src_temp0_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| } |
| //updating top flag |
| _mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 16; |
| } |
| |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy)); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| for(row = ht; row >= 4; row -= 4) |
| { |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //row1 -row2 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //packing row 0 n row 1 |
| src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b); |
| //row = 3 |
| src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| // row = 4 |
| src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 4 * src_strd)); |
| |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| signdwn1_16x8b = _mm_slli_si128(signdwn1_16x8b, 8); //allign left (1-2) |
| //separating +ve and and -ve values.(2,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_bottom_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signdwn1_16x8b, 8); //(2-3) ,(1-2) (substract with down) |
| edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); |
| //separating +ve and and -ve values.(3,4) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_top_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-4) |
| //combining sign-left and sign_right |
| edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-4),(2-3) |
| |
| edge1_16x8b = _mm_sub_epi8(edge1_16x8b, signup0_16x8b); //(3,2) |
| |
| //packing row 2 n row 3 |
| src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b); |
| //for the next iteration signup0_16x8b = -signdwn1_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); //(4-3) |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //the next top already in src_top_16x8b |
| //src_top_16x8b = src_temp1_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, cmp_lt0_16x8b); |
| src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| //row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b); |
| |
| src_temp0_16x8b = src_temp1_16x8b; |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| pu1_src_cpy += (src_strd << 2); |
| |
| } |
| ht_rem = ht & 0x2; |
| if(ht_rem) |
| { |
| |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //row1 -row2 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| //for the next iteration signup0_16x8b = -signdwn1_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); //(2-1) for next |
| src_top_16x8b = src_temp1_16x8b; |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //the next top already in src_top_16x8b |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| |
| } |
| ht_rem = ht & 0x1; |
| if(ht_rem) |
| { |
| |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); |
| edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| src_top_16x8b = src_temp0_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| } |
| _mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 8; |
| } |
| } |
| } |
| |
| void ihevc_sao_edge_offset_class1_chroma_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| UWORD8 *pu1_src_top_right, |
| UWORD8 *pu1_src_bot_left, |
| UWORD8 *pu1_avail, |
| WORD8 *pi1_sao_offset_u, |
| WORD8 *pi1_sao_offset_v, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_top_cpy; |
| UWORD8 *pu1_src_cpy; |
| WORD32 wd_rem; |
| |
| |
| __m128i src_top_16x8b, src_bottom_16x8b; |
| __m128i src_temp0_16x8b, src_temp1_16x8b; |
| __m128i signup0_16x8b, signdwn1_16x8b; |
| __m128i cmp_gt0_16x8b, cmp_lt0_16x8b; |
| __m128i edge0_16x8b, edge1_16x8b; |
| __m128i edge_idx_8x16b, sao_offset_8x16b; |
| __m128i const2_16x8b, const0_16x8b; |
| __m128i chroma_offset_8x16b; |
| |
| UNUSED(pu1_src_top_right); |
| UNUSED(pu1_src_bot_left); |
| |
| /* Updating left and top and top-left */ |
| for(row = 0; row < ht; row++) |
| { |
| pu1_src_left[2 * row] = pu1_src[row * src_strd + (wd - 2)]; |
| pu1_src_left[2 * row + 1] = pu1_src[row * src_strd + (wd - 1)]; |
| } |
| pu1_src_top_left[0] = pu1_src_top[wd - 2]; |
| pu1_src_top_left[1] = pu1_src_top[wd - 1]; |
| |
| |
| |
| pu1_src_top_cpy = pu1_src_top; |
| edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx); |
| sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u); |
| const0_16x8b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v); |
| chroma_offset_8x16b = _mm_set1_epi16(0x0800); |
| /* Update height and source pointers based on the availability flags */ |
| if(0 == pu1_avail[2]) |
| { |
| pu1_src_top_cpy = pu1_src; |
| pu1_src += src_strd; |
| ht--; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| ht--; |
| } |
| sao_offset_8x16b = _mm_unpacklo_epi64(sao_offset_8x16b, const0_16x8b); |
| const2_16x8b = _mm_set1_epi8(2); |
| const0_16x8b = _mm_setzero_si128(); |
| |
| |
| { |
| WORD32 ht_rem; |
| |
| |
| |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| for(row = ht; row >= 2; row -= 2) |
| { |
| |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); |
| |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b); |
| |
| //for the next iteration signup0_16x8b = -signdwn1_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //copying the next top |
| src_top_16x8b = src_temp1_16x8b; |
| |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b); |
| |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| } |
| ht_rem = ht & 0x1; |
| |
| if(ht_rem) |
| { |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| //current row -next row |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and botton and constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| //copying the next top |
| src_top_16x8b = src_temp0_16x8b; |
| |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| } |
| //updating top flag |
| _mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 16; |
| } |
| |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy)); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| for(row = ht; row >= 4; row -= 4) |
| { |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //row1 -row2 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //packing row 0 n row 1 |
| src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b); |
| //row = 3 |
| src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| // row = 4 |
| src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 4 * src_strd)); |
| |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| signdwn1_16x8b = _mm_slli_si128(signdwn1_16x8b, 8); //allign left (1-2) |
| //separating +ve and and -ve values.(2,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_bottom_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signdwn1_16x8b, 8); //(2-3) ,(1-2) (substract with down) |
| edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); |
| //separating +ve and and -ve values.(3,4) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_top_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-4) |
| //combining sign-left and sign_right |
| edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-4),(2-3) |
| |
| edge1_16x8b = _mm_sub_epi8(edge1_16x8b, signup0_16x8b); //(3,2) |
| |
| //packing row 2 n row 3 |
| src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b); |
| //for the next iteration signup0_16x8b = -signdwn1_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); //(4-3) |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //the next top already in src_top_16x8b |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, edge1_16x8b); |
| src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| //row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b); |
| |
| src_temp0_16x8b = src_temp1_16x8b; |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| pu1_src_cpy += (src_strd << 2); |
| |
| } |
| ht_rem = ht & 0x2; |
| if(ht_rem) |
| { |
| |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_temp1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //row1 -row2 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| //for the next iteration signup0_16x8b = -signdwn1_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_lt0_16x8b, cmp_gt0_16x8b); //(2-1) for next |
| src_top_16x8b = src_temp1_16x8b; |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| //the next top already in src_top_16x8b |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| |
| } |
| ht_rem = ht & 0x1; |
| if(ht_rem) |
| { |
| |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_bottom_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| src_top_16x8b = src_temp0_16x8b; |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); |
| edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| } |
| _mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 8; |
| } |
| } |
| } |
| |
| /* 135 degree filtering */ |
| void ihevc_sao_edge_offset_class2_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| UWORD8 *pu1_src_top_right, |
| UWORD8 *pu1_src_bot_left, |
| UWORD8 *pu1_avail, |
| WORD8 *pi1_sao_offset, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_top_cpy, *pu1_src_left_cpy, *pu1_src_left_cpy2; |
| UWORD8 *pu1_left_tmp, *pu1_src_left_str, *pu1_src_left_str2; |
| UWORD8 *pu1_firstleft; |
| UWORD8 *pu1_src_cpy, *pu1_src_org; |
| UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy; |
| UWORD8 au1_src_left_tmp[MAX_CTB_SIZE + 8]; |
| UWORD8 au1_src_left_tmp1[MAX_CTB_SIZE + 8]; |
| WORD32 wd_rem; |
| UWORD8 u1_pos_0_0_tmp, u1_pos_wd_ht_tmp; |
| WORD32 ht_tmp, ht_0; |
| |
| WORD32 bit_depth; |
| UWORD8 u1_avail0, u1_avail1; |
| |
| __m128i src_top_16x8b, src_bottom_16x8b; |
| __m128i src_temp0_16x8b, src_temp1_16x8b; |
| __m128i signup0_16x8b, signdwn1_16x8b; |
| __m128i cmp_gt0_16x8b, cmp_lt0_16x8b; |
| __m128i edge0_16x8b, edge1_16x8b; |
| __m128i au1_mask8x16b; |
| __m128i edge_idx_8x16b, sao_offset_8x16b; |
| __m128i const2_16x8b, const0_16x8b; |
| __m128i left_store_16x8b; |
| UNUSED(pu1_src_top_right); |
| UNUSED(pu1_src_bot_left); |
| |
| ht_0 = ht; ht_tmp = ht; |
| au1_mask8x16b = _mm_set1_epi8(0xff); |
| |
| //setting availability mask to ff size MAX_CTB_SIZE |
| for(col = 0; col < MAX_CTB_SIZE; col += 16) |
| _mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b); |
| for(row = 0; row < ht; row++) |
| { |
| au1_src_left_tmp[row] = pu1_src_left[row]; |
| } |
| bit_depth = BIT_DEPTH_LUMA; |
| pu1_src_org = pu1_src; |
| pu1_src_top_cpy = pu1_src_top; |
| pu1_src_left_cpy2 = au1_src_left_tmp; |
| pu1_src_left_cpy = au1_src_left_tmp; |
| pu1_src_left_str2 = au1_src_left_tmp1; |
| pu1_src_left_str = au1_src_left_tmp1; |
| edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx); |
| sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset); |
| |
| |
| /* If top-left is available, process separately */ |
| if(0 != pu1_avail[4]) |
| { |
| WORD8 edge_idx; |
| |
| edge_idx = 2 + SIGN(pu1_src[0] - pu1_src_top_left[0]) + |
| SIGN(pu1_src[0] - pu1_src[1 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_0_0_tmp = CLIP3(pu1_src[0] + pi1_sao_offset[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_0_0_tmp = pu1_src[0]; |
| } |
| } |
| else |
| { |
| u1_pos_0_0_tmp = pu1_src[0]; |
| } |
| |
| /* If bottom-right is available, process separately */ |
| if(0 != pu1_avail[7]) |
| { |
| WORD8 edge_idx; |
| |
| edge_idx = 2 + SIGN(pu1_src[wd - 1 + (ht - 1) * src_strd] - pu1_src[wd - 1 + (ht - 1) * src_strd - 1 - src_strd]) + |
| SIGN(pu1_src[wd - 1 + (ht - 1) * src_strd] - pu1_src[wd - 1 + (ht - 1) * src_strd + 1 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_wd_ht_tmp = CLIP3(pu1_src[wd - 1 + (ht - 1) * src_strd] + pi1_sao_offset[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_wd_ht_tmp = pu1_src[wd - 1 + (ht - 1) * src_strd]; |
| } |
| } |
| else |
| { |
| u1_pos_wd_ht_tmp = pu1_src[wd - 1 + (ht - 1) * src_strd]; |
| } |
| pu1_firstleft = pu1_src_top_left; |
| |
| /* Update height and source pointers based on the availability flags */ |
| if(0 == pu1_avail[2]) |
| { |
| pu1_firstleft = pu1_src_left_cpy2; |
| pu1_src_left_cpy2++; |
| pu1_src_left_str2++; |
| pu1_src_top_cpy = pu1_src; |
| pu1_src += src_strd; |
| ht--; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| ht--; |
| ht_0--; |
| } |
| //storing top left in a mmx register |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_firstleft); |
| const2_16x8b = _mm_set1_epi8(2); |
| const0_16x8b = _mm_setzero_si128(); |
| left_store_16x8b = _mm_slli_si128(left_store_16x8b, 15); |
| //update top -left |
| *pu1_src_top_left = pu1_src_top[wd - 1]; |
| //availability mask creation |
| u1_avail0 = pu1_avail[0]; |
| u1_avail1 = pu1_avail[1]; |
| au1_mask[0] = u1_avail0; |
| au1_mask[wd - 1] = u1_avail1; |
| { |
| WORD32 ht_rem; |
| |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| au1_mask_cpy = au1_mask; |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| src_top_16x8b = _mm_alignr_epi8(src_top_16x8b, left_store_16x8b, 15); |
| //loading the mask |
| au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| |
| for(row = ht; row >= 2; row -= 2) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 1 right |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 1)); |
| //to insert left in row 0 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 15); |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 15); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(0-1) |
| //row1-row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| // row = 2 right |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd + 1)); |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-0) |
| |
| |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| |
| //storing the row 1 left for next row. |
| signup0_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| |
| //combining sign-left and sign_right |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b); |
| //manipulation for bottom - row 1 |
| signup0_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signup0_16x8b, 15); |
| //eliminating old left for row 0 and row 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| //bottom - row1 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signup0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signup0_16x8b, src_bottom_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //for the next iteration bottom -row1 |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row1 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 15); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //row0 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| //copying the next top |
| src_top_16x8b = src_temp1_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| |
| //store left boundary |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b); |
| |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 2; |
| pu1_src_left_str += 2; |
| } |
| ht_rem = ht & 0x1; |
| |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 1)); |
| //current row -next row |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and botton and constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| //eliminating old left for row 0 and row 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1); |
| |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //row0 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| //copying the next top |
| src_top_16x8b = src_temp0_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| //store left boundary |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| pu1_src_left_cpy += 1; |
| pu1_src_left_str += 1; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| pu1_src_left_str[0] = pu1_src_cpy[15]; |
| } |
| if(0 == pu1_avail[2]) |
| { |
| pu1_src_left_str[-ht_0] = pu1_src[15 - src_strd]; |
| } |
| |
| //for the top left of next part of the block |
| left_store_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //updating top flag |
| _mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 16; |
| au1_mask_cpy += 16; |
| |
| |
| pu1_left_tmp = pu1_src_left_cpy2; |
| pu1_src_left_cpy2 = pu1_src_left_str2; |
| pu1_src_left_str2 = pu1_left_tmp; |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| } |
| |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy)); |
| src_top_16x8b = _mm_alignr_epi8(src_top_16x8b, left_store_16x8b, 15); |
| au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); //???? |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //preparing au1_mask |
| au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| |
| for(row = ht; row >= 4; row -= 4) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| //right row1 |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1); |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //manipulation for row 1 -row 0 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 15); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row 0 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 15); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row 1 -row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //right row2 |
| signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 1); |
| //packing row 0 n row 1 |
| src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b); |
| //row1 -row2 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //manipulation for row 2 -row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //row 1 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15); |
| //row = 3 |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| |
| // row = 4 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 4 * src_strd)); |
| |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| |
| //separating +ve and and -ve values.(2,1) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| //manipulation for row 3 -row 2 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 13); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row 2 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) |
| |
| //separating +ve and and -ve values.(3,2) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); //aligned left (2-1) |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //right row3 |
| signdwn1_16x8b = _mm_srli_si128(src_top_16x8b, 1); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-2) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(3-2) ,(2-1) |
| |
| //separating +ve and and -ve values.(2,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| //right row 4 |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3) |
| |
| //separating +ve and and -ve values.(3,bottom) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); //aligned left (2-3) |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-bottom) |
| edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-bottom),(2-3) |
| |
| //manipulation for bottom -row 3 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| //eliminating old left for row 0,1,2,3 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4); |
| //packing row 2 n row 3 |
| src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b); |
| //row 3 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_top_16x8b, signdwn1_16x8b, 15); |
| //loading row 3 right into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_bottom_16x8b, 15); |
| //adding bottom and top values of row 2 and row 3 |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signup0_16x8b); //(3,2) |
| //separating +ve and and -ve values.(botttom,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //to store right of row 2 |
| signdwn1_16x8b = _mm_slli_si128(src_bottom_16x8b, 8); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(bottom -3) for next iteration |
| |
| //storing right of row 2into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| //to store right of row 0 |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //storing right of row 1 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| |
| //storing right of row 0 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, cmp_lt0_16x8b); |
| src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| //row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b); |
| |
| src_temp0_16x8b = src_temp1_16x8b; |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| pu1_src_cpy += (src_strd << 2); |
| pu1_src_left_cpy += 4; |
| pu1_src_left_str += 4; |
| } |
| ht_rem = ht & 0x2; |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //row 0 -row 1 |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //manipulation for row 1 -row 0 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 15); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 15); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //row1-row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign chang |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row 1 -bottom |
| signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 1); |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //manipulation for bottom -row1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //manipulation for bottom- row 1 |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| //bottom - row 1 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| |
| //eliminating old left for row 0,1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| signdwn1_16x8b = _mm_slli_si128(src_temp1_16x8b, 8); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //for the next iteration signup0_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) for next |
| |
| //storing right of row 1 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| //for storing right of row 1 |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| |
| src_top_16x8b = src_temp1_16x8b; |
| //storing right of row 0 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //the next top already in src_top_16x8b |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 2; |
| pu1_src_left_str += 2; |
| } |
| ht_rem = ht & 0x1; |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| //left store manipulation 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1); |
| //row 0 -row1 |
| signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 1); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| //for row 0 right to put into left store |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); |
| edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8); |
| //filling the left boundary value |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| src_top_16x8b = src_temp0_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| pu1_src_left_cpy += 1; |
| pu1_src_left_str += 1; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| pu1_src_left_str[0] = pu1_src_cpy[7]; |
| } |
| |
| if(0 == pu1_avail[2]) |
| { |
| pu1_src_left_str[-ht_0] = pu1_src[7 - src_strd]; |
| } |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 8; |
| au1_mask_cpy += 16; |
| |
| pu1_left_tmp = pu1_src_left_cpy2; |
| pu1_src_left_cpy2 = pu1_src_left_str2; |
| pu1_src_left_str2 = pu1_left_tmp; |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| } |
| pu1_src_org[0] = u1_pos_0_0_tmp; |
| pu1_src_org[wd - 1 + (ht_tmp - 1) * src_strd] = u1_pos_wd_ht_tmp; |
| pu1_src_left_cpy = (0 == pu1_avail[2]) ? (pu1_src_left_cpy - 1) : pu1_src_left_cpy; |
| for(row = 0; row < ht_tmp; row++) |
| { |
| pu1_src_left[row] = pu1_src_left_cpy[row]; |
| } |
| } |
| |
| } |
| |
| /* 135 degree filtering */ |
| void ihevc_sao_edge_offset_class2_chroma_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| UWORD8 *pu1_src_top_right, |
| UWORD8 *pu1_src_bot_left, |
| UWORD8 *pu1_avail, |
| WORD8 *pi1_sao_offset_u, |
| WORD8 *pi1_sao_offset_v, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_top_cpy, *pu1_src_left_cpy, *pu1_src_left_cpy2; |
| UWORD8 *pu1_left_tmp, *pu1_src_left_str, *pu1_src_left_str2; |
| UWORD8 *pu1_firstleft; |
| UWORD8 *pu1_src_cpy, *pu1_src_org; |
| UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy; |
| UWORD8 au1_src_left_tmp[2 * (MAX_CTB_SIZE + 8)]; |
| UWORD8 au1_src_left_tmp1[2 * (MAX_CTB_SIZE + 8)]; |
| WORD32 wd_rem; |
| UWORD8 u1_pos_0_0_tmp_u, u1_pos_0_0_tmp_v, u1_pos_wd_ht_tmp_u, u1_pos_wd_ht_tmp_v; |
| WORD32 ht_tmp; |
| WORD32 ht_0; |
| |
| WORD32 bit_depth; |
| UWORD8 u1_avail0, u1_avail1; |
| |
| __m128i src_temp0_16x8b, src_temp1_16x8b; |
| __m128i signup0_16x8b, signdwn1_16x8b; |
| __m128i cmp_gt0_16x8b, cmp_lt0_16x8b; |
| __m128i edge0_16x8b, edge1_16x8b; |
| __m128i src_top_16x8b, src_bottom_16x8b; |
| __m128i au1_mask8x16b; |
| __m128i edge_idx_8x16b, sao_offset_8x16b; |
| __m128i const2_16x8b, const0_16x8b; |
| __m128i left_store_16x8b; |
| __m128i chroma_offset_8x16b; |
| |
| UNUSED(pu1_src_top_right); |
| UNUSED(pu1_src_bot_left); |
| |
| ht_0 = ht; ht_tmp = ht; |
| au1_mask8x16b = _mm_set1_epi8(0xff); |
| /* Updating left and top-left */ |
| for(row = 0; row < 2 * ht; row++) |
| { |
| au1_src_left_tmp[row] = pu1_src_left[row]; |
| } |
| //setting availability mask to ff size MAX_CTB_SIZE |
| for(col = 0; col < MAX_CTB_SIZE; col += 16) |
| _mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b); |
| bit_depth = BIT_DEPTH_LUMA; |
| pu1_src_org = pu1_src; |
| pu1_src_top_cpy = pu1_src_top; |
| pu1_src_left_cpy2 = au1_src_left_tmp; |
| pu1_src_left_cpy = au1_src_left_tmp; |
| pu1_src_left_str2 = au1_src_left_tmp1; |
| pu1_src_left_str = au1_src_left_tmp1; |
| edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx); |
| sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u); |
| const0_16x8b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v); |
| chroma_offset_8x16b = _mm_set1_epi16(0x0800); |
| |
| /* If top-left is available, process separately */ |
| if(0 != pu1_avail[4]) |
| { |
| WORD32 edge_idx; |
| |
| /* U */ |
| edge_idx = 2 + SIGN(pu1_src[0] - pu1_src_top_left[0]) + |
| SIGN(pu1_src[0] - pu1_src[2 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_0_0_tmp_u = CLIP3(pu1_src[0] + pi1_sao_offset_u[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_0_0_tmp_u = pu1_src[0]; |
| } |
| |
| /* V */ |
| edge_idx = 2 + SIGN(pu1_src[1] - pu1_src_top_left[1]) + |
| SIGN(pu1_src[1] - pu1_src[1 + 2 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_0_0_tmp_v = CLIP3(pu1_src[1] + pi1_sao_offset_v[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_0_0_tmp_v = pu1_src[1]; |
| } |
| } |
| else |
| { |
| u1_pos_0_0_tmp_u = pu1_src[0]; |
| u1_pos_0_0_tmp_v = pu1_src[1]; |
| } |
| |
| /* If bottom-right is available, process separately */ |
| if(0 != pu1_avail[7]) |
| { |
| WORD32 edge_idx; |
| |
| /* U */ |
| edge_idx = 2 + SIGN(pu1_src[wd - 2 + (ht - 1) * src_strd] - pu1_src[wd - 2 + (ht - 1) * src_strd - 2 - src_strd]) + |
| SIGN(pu1_src[wd - 2 + (ht - 1) * src_strd] - pu1_src[wd - 2 + (ht - 1) * src_strd + 2 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_wd_ht_tmp_u = CLIP3(pu1_src[wd - 2 + (ht - 1) * src_strd] + pi1_sao_offset_u[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_wd_ht_tmp_u = pu1_src[wd - 2 + (ht - 1) * src_strd]; |
| } |
| |
| /* V */ |
| edge_idx = 2 + SIGN(pu1_src[wd - 1 + (ht - 1) * src_strd] - pu1_src[wd - 1 + (ht - 1) * src_strd - 2 - src_strd]) + |
| SIGN(pu1_src[wd - 1 + (ht - 1) * src_strd] - pu1_src[wd - 1 + (ht - 1) * src_strd + 2 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_wd_ht_tmp_v = CLIP3(pu1_src[wd - 1 + (ht - 1) * src_strd] + pi1_sao_offset_v[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_wd_ht_tmp_v = pu1_src[wd - 1 + (ht - 1) * src_strd]; |
| } |
| } |
| else |
| { |
| u1_pos_wd_ht_tmp_u = pu1_src[wd - 2 + (ht - 1) * src_strd]; |
| u1_pos_wd_ht_tmp_v = pu1_src[wd - 1 + (ht - 1) * src_strd]; |
| } |
| pu1_firstleft = pu1_src_top_left; |
| |
| /* Update height and source pointers based on the availability flags */ |
| if(0 == pu1_avail[2]) |
| { |
| pu1_firstleft = pu1_src_left_cpy2; |
| pu1_src_left_cpy2 += 2; |
| pu1_src_left_str2 += 2; |
| pu1_src_top_cpy = pu1_src; |
| pu1_src += src_strd; |
| ht--; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| ht--; |
| ht_0--; |
| } |
| //storing top left in a mmx register |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_firstleft); |
| sao_offset_8x16b = _mm_unpacklo_epi64(sao_offset_8x16b, const0_16x8b); |
| const2_16x8b = _mm_set1_epi8(2); |
| const0_16x8b = _mm_setzero_si128(); |
| left_store_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| |
| //availability mask creation |
| u1_avail0 = pu1_avail[0]; |
| u1_avail1 = pu1_avail[1]; |
| au1_mask[0] = u1_avail0; |
| au1_mask[1] = u1_avail0; |
| au1_mask[wd - 1] = u1_avail1; |
| au1_mask[wd - 2] = u1_avail1; |
| |
| /* top-left arrays */ |
| pu1_src_top_left[0] = pu1_src_top[wd - 2]; |
| pu1_src_top_left[1] = pu1_src_top[wd - 1]; |
| { |
| WORD32 ht_rem; |
| au1_mask_cpy = au1_mask; |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| src_top_16x8b = _mm_alignr_epi8(src_top_16x8b, left_store_16x8b, 14); |
| //loading the mask |
| au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| |
| for(row = ht; row >= 2; row -= 2) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 1 right |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 2)); |
| //to insert left in row 0 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 14); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(0-1) |
| //row1-row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| // row = 2 right |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd + 2)); |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-0) |
| |
| |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| |
| //storing the row 1 left for next row. |
| signup0_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| |
| //combining sign-left and sign_right |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b); |
| //manipulation for bottom - row 1 |
| signup0_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signup0_16x8b, 14); |
| //eliminating old left for row 0 and row 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4); |
| //bottom - row1 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signup0_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signup0_16x8b, src_bottom_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //for the next iteration bottom -row1 |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row1 getting it right for left of next iteration |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 14); |
| //copying the next top |
| src_top_16x8b = src_temp1_16x8b; |
| //row0 getting its right for left of next iteration. |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b); |
| |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| |
| //store left boundary |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b); |
| |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 4; |
| pu1_src_left_str += 4; |
| } |
| ht_rem = ht & 0x1; |
| |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 2)); |
| //current row -next row |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and botton and constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| |
| //eliminating old left for row 0 and row 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| //copying the next top |
| src_top_16x8b = src_temp0_16x8b; |
| //row0 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| pu1_src_left_cpy += 2; |
| pu1_src_left_str += 2; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| pu1_src_left_str[1] = pu1_src_cpy[15]; |
| pu1_src_left_str[0] = pu1_src_cpy[14]; |
| } |
| if(0 == pu1_avail[2]) |
| { |
| pu1_src_left_str[-2 * ht_0] = pu1_src[14 - src_strd]; |
| pu1_src_left_str[-2 * ht_0 + 1] = pu1_src[15 - src_strd]; |
| } |
| |
| //for the top left of next part of the block |
| left_store_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //updating top flag |
| _mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 16; |
| au1_mask_cpy += 16; |
| |
| pu1_left_tmp = pu1_src_left_cpy2; |
| pu1_src_left_cpy2 = pu1_src_left_str2; |
| pu1_src_left_str2 = pu1_left_tmp; |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| } |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_cpy)); |
| src_top_16x8b = _mm_alignr_epi8(src_top_16x8b, left_store_16x8b, 14); |
| au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); //???? |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //preparing au1_mask |
| au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| |
| for(row = ht; row >= 4; row -= 4) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| //right row1 |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2); |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //manipulation for row 1 -row 0 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row 0 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 14); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row 1 -row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //right row2 |
| signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 2); |
| //packing row 0 n row 1 |
| src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b); |
| //row1 -row2 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //manipulation for row 2 -row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| //row 1 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14); |
| //row = 3 |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| |
| // row = 4 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 4 * src_strd)); |
| |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| |
| //separating +ve and and -ve values.(2,1) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| //manipulation for row 3 -row 2 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 10); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row 2 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) |
| |
| //separating +ve and and -ve values.(3,2) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); //aligned left (2-1) |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //right row3 |
| signdwn1_16x8b = _mm_srli_si128(src_top_16x8b, 2); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-2) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(3-2) ,(2-1) |
| |
| //separating +ve and and -ve values.(2,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| //right row 4 |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3) |
| |
| //separating +ve and and -ve values.(3,bottom) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); //aligned left (2-3) |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-bottom) |
| edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-bottom),(2-3) |
| |
| //manipulation for bottom -row 3 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 8); |
| //eliminating old left for row 0,1,2,3 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 8); |
| //packing row 2 n row 3 |
| src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b); |
| //row 3 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_top_16x8b, signdwn1_16x8b, 14); |
| |
| //adding bottom and top values of row 2 and row 3 |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signup0_16x8b); //(3,2) |
| //separating +ve and and -ve values.(botttom,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(bottom -3) for next iteration |
| |
| //to store right of row 2 |
| signdwn1_16x8b = _mm_slli_si128(src_bottom_16x8b, 8); |
| //loading row 3 right into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_bottom_16x8b, 14); |
| //storing right of row 2into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| //to store right of row 0 |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //storing right of row 1 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| //storing right of row 0 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, edge1_16x8b); |
| src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8); |
| |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| //row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b); |
| |
| src_temp0_16x8b = src_temp1_16x8b; |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| pu1_src_cpy += (src_strd << 2); |
| pu1_src_left_cpy += 8; |
| pu1_src_left_str += 8; |
| } |
| ht_rem = ht & 0x2; |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //row 0 -row 1 |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //manipulation for row 1 -row 0 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp0_16x8b, signdwn1_16x8b, 14); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //row1-row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign chang |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row 1 -bottom |
| signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 2); |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //manipulation for bottom -row1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| //eliminating old left for row 0,1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4); |
| //manipulation for bottom- row 1 |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| //bottom - row 1 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| |
| //shifting row 1 |
| signdwn1_16x8b = _mm_slli_si128(src_temp1_16x8b, 8); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //for the next iteration signup0_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) for next |
| //storing right of row 1 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); //for storing right of row 0 |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //the next top in src_top_16x8b |
| src_top_16x8b = src_temp1_16x8b; |
| //storing right of row 0 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| //the next top already in src_top_16x8b |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 4; |
| pu1_src_left_str += 4; |
| } |
| ht_rem = ht & 0x1; |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| //row 0 -row1 |
| signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 2); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| |
| //for row 0 right to put into left store |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //left store manipulation 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| src_top_16x8b = src_temp0_16x8b; |
| //filling the left boundary value |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); |
| edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8); |
| |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| pu1_src_left_cpy += 2; |
| pu1_src_left_str += 2; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| pu1_src_left_str[1] = pu1_src_cpy[7]; |
| pu1_src_left_str[0] = pu1_src_cpy[6]; |
| } |
| |
| if(0 == pu1_avail[2]) |
| { |
| pu1_src_left_str[-2 * ht_0] = pu1_src[6 - src_strd]; |
| pu1_src_left_str[-2 * ht_0 + 1] = pu1_src[7 - src_strd]; |
| } |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 8; |
| |
| pu1_left_tmp = pu1_src_left_cpy2; |
| pu1_src_left_cpy2 = pu1_src_left_str2; |
| pu1_src_left_str2 = pu1_left_tmp; |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| } |
| pu1_src_org[0] = u1_pos_0_0_tmp_u; |
| pu1_src_org[1] = u1_pos_0_0_tmp_v; |
| pu1_src_org[wd - 2 + (ht_tmp - 1) * src_strd] = u1_pos_wd_ht_tmp_u; |
| pu1_src_org[wd - 1 + (ht_tmp - 1) * src_strd] = u1_pos_wd_ht_tmp_v; |
| pu1_src_left_cpy = (0 == pu1_avail[2]) ? (pu1_src_left_cpy - 2) : pu1_src_left_cpy; |
| for(row = 0; row < 2 * ht_tmp; row++) |
| { |
| pu1_src_left[row] = pu1_src_left_cpy[row]; |
| } |
| } |
| |
| } |
| |
| void ihevc_sao_edge_offset_class3_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| UWORD8 *pu1_src_top_right, |
| UWORD8 *pu1_src_bot_left, |
| UWORD8 *pu1_avail, |
| WORD8 *pi1_sao_offset, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_top_cpy, *pu1_src_left_cpy, *pu1_src_left_cpy2; |
| UWORD8 *pu1_left_tmp, *pu1_src_left_str, *pu1_src_left_str2; |
| UWORD8 *pu1_src_cpy, *pu1_src_org; |
| UWORD8 au1_src_left_tmp[MAX_CTB_SIZE + 8]; |
| UWORD8 au1_src_left_tmp1[MAX_CTB_SIZE + 8]; |
| UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy; |
| WORD32 wd_rem; |
| UWORD8 u1_pos_wd_0_tmp, u1_pos_0_ht_tmp; |
| WORD32 ht_tmp; |
| WORD32 bit_depth; |
| UWORD8 u1_avail0, u1_avail1; |
| |
| __m128i src_top_16x8b, src_bottom_16x8b; |
| __m128i src_temp0_16x8b, src_temp1_16x8b; |
| __m128i signup0_16x8b, signdwn1_16x8b; |
| __m128i cmp_gt0_16x8b, cmp_lt0_16x8b; |
| __m128i edge0_16x8b, edge1_16x8b; |
| __m128i au1_mask8x16b; |
| __m128i edge_idx_8x16b, sao_offset_8x16b; |
| __m128i const2_16x8b, const0_16x8b; |
| __m128i left_store_16x8b; |
| |
| ht_tmp = ht; |
| au1_mask8x16b = _mm_set1_epi8(0xff); |
| |
| au1_src_left_tmp[0] = pu1_src[(wd - 1)]; |
| //manipulation for bottom left |
| for(row = 1; row < ht; row++) |
| { |
| au1_src_left_tmp[row] = pu1_src_left[row]; |
| } |
| au1_src_left_tmp[ht] = pu1_src_bot_left[0]; |
| |
| *pu1_src_top_left = pu1_src_top[wd - 1]; |
| //setting availability mask to ff size MAX_CTB_SIZE |
| for(col = 0; col < MAX_CTB_SIZE; col += 16) |
| _mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b); |
| bit_depth = BIT_DEPTH_LUMA; |
| pu1_src_org = pu1_src; |
| pu1_src_top_cpy = pu1_src_top; |
| pu1_src_left_cpy2 = au1_src_left_tmp; |
| pu1_src_left_cpy = au1_src_left_tmp; |
| pu1_src_left_str2 = au1_src_left_tmp1; |
| pu1_src_left_str = au1_src_left_tmp1; |
| edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx); |
| sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset); |
| |
| /* If top-right is available, process separately */ |
| if(0 != pu1_avail[5]) |
| { |
| WORD32 edge_idx; |
| |
| edge_idx = 2 + SIGN(pu1_src[wd - 1] - pu1_src_top_right[0]) + |
| SIGN(pu1_src[wd - 1] - pu1_src[wd - 1 - 1 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_wd_0_tmp = CLIP3(pu1_src[wd - 1] + pi1_sao_offset[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_wd_0_tmp = pu1_src[wd - 1]; |
| } |
| } |
| else |
| { |
| u1_pos_wd_0_tmp = pu1_src[wd - 1]; |
| } |
| |
| /* If bottom-left is available, process separately */ |
| if(0 != pu1_avail[6]) |
| { |
| WORD32 edge_idx; |
| |
| edge_idx = 2 + SIGN(pu1_src[(ht - 1) * src_strd] - pu1_src[(ht - 1) * src_strd + 1 - src_strd]) + |
| SIGN(pu1_src[(ht - 1) * src_strd] - pu1_src_bot_left[0]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_0_ht_tmp = CLIP3(pu1_src[(ht - 1) * src_strd] + pi1_sao_offset[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_0_ht_tmp = pu1_src[(ht - 1) * src_strd]; |
| } |
| } |
| else |
| { |
| u1_pos_0_ht_tmp = pu1_src[(ht - 1) * src_strd]; |
| } |
| |
| |
| |
| /* Update height and source pointers based on the availability flags */ |
| if(0 == pu1_avail[2]) |
| { |
| pu1_src_left_cpy2++; |
| pu1_src_left_str2++; |
| pu1_src_top_cpy = pu1_src; |
| pu1_src += src_strd; |
| ht--; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| ht--; |
| } |
| |
| |
| const2_16x8b = _mm_set1_epi8(2); |
| const0_16x8b = _mm_setzero_si128(); |
| |
| |
| //availability mask creation |
| u1_avail0 = pu1_avail[0]; |
| u1_avail1 = pu1_avail[1]; |
| au1_mask[0] = u1_avail0; |
| au1_mask[wd - 1] = u1_avail1; |
| { |
| WORD32 ht_rem; |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| au1_mask_cpy = au1_mask; |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col + 1)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| |
| //loading the mask |
| au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| for(row = ht; row >= 2; row -= 2) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy)); |
| //row = 1 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| //to insert left in row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| // row = 0 right |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 1)); |
| |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15); |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(0-1) |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| |
| //row1-row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| // row = 1 right |
| signdwn1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 1)); |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-0) |
| |
| //bottom - row1 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //for the next iteration bottom -row1 |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //to insert left in row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 13); |
| //manipulation for row 1 - bottom |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15); |
| |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //combining sign-left and sign_right |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b); |
| |
| //eliminating old left for row 0 and row 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| |
| //row1 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 15); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //row0 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| //copying the next top |
| src_top_16x8b = src_temp1_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| //store left boundary |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b); |
| |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 2; |
| pu1_src_left_str += 2; |
| } |
| ht_rem = ht & 0x1; |
| |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| //to insert left in row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15); |
| |
| //current row -next row |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and bottom and constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| //eliminating old left for row 0 and row 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1); |
| |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //row0 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| //copying the next top |
| src_top_16x8b = src_temp0_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| //store left boundary |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_left_cpy++; |
| pu1_src_left_str++; |
| } |
| { //for bottom right |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| } |
| //for the top left of next part of the block |
| left_store_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //updating top flag |
| _mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 16; |
| au1_mask_cpy += 16; |
| |
| pu1_left_tmp = pu1_src_left_cpy2; |
| pu1_src_left_cpy2 = pu1_src_left_str2; |
| pu1_src_left_str2 = pu1_left_tmp; |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| } |
| |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| pu1_src_cpy = pu1_src; |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col + 1)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); //???? |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //preparing au1_mask |
| au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| |
| for(row = ht; row >= 4; row -= 4) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| //manipulation for row 0 -row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //row 1 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15); |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulatiing for row 1 -row 0 |
| signdwn1_16x8b = _mm_srli_si128(src_temp0_16x8b, 1); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row 1 -row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //manipulation for row 1 -row 2 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 13); |
| //row 2 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15); |
| //packing row 0 n row 1 |
| src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b); |
| //row1 -row2 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| |
| //row 1 right |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1); |
| //row = 3 |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| |
| // row = 4 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 4 * src_strd)); |
| |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| |
| //separating +ve and and -ve values.(2,1) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row 2 right |
| signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 1); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) |
| |
| //separating +ve and and -ve values.(3,2) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); //aligned left (2-1) |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulation for row 2 -row 3 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| //row 3 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_top_16x8b, signdwn1_16x8b, 15); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-2) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(3-2) ,(2-1) |
| |
| //separating +ve and and -ve values.(2,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| |
| //manipulation for row 3 -bottom |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 11); |
| //bottom left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3) |
| |
| //separating +ve and and -ve values.(3,bottom) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); //aligned left (2-3) |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-bottom) |
| edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-bottom),(2-3) |
| |
| |
| //eliminating old left for row 0,1,2,3 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4); |
| //packing row 2 n row 3 |
| src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b); |
| //row 3 right |
| signdwn1_16x8b = _mm_srli_si128(src_top_16x8b, 1); |
| //loading row 3 right into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_bottom_16x8b, 15); |
| //adding bottom and top values of row 2 and row 3 |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signup0_16x8b); //(3,2) |
| //separating +ve and and -ve values.(botttom,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //to store right of row 2 |
| signdwn1_16x8b = _mm_slli_si128(src_bottom_16x8b, 8); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(bottom -3) for next iteration |
| |
| //storing right of row 2into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| //to store right of row 0 |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //storing right of row 1 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| |
| //storing right of row 0 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, cmp_lt0_16x8b); |
| src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| //row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b); |
| |
| src_temp0_16x8b = src_temp1_16x8b; |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| pu1_src_cpy += (src_strd << 2); |
| pu1_src_left_cpy += 4; |
| pu1_src_left_str += 4; |
| } |
| ht_rem = ht & 0x2; |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //manipulation for row 0 -row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //bottom left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 15); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_srli_si128(src_temp0_16x8b, 1); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //row1-row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign chang |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //manipulation for row 1 -bottom |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 13); |
| //bottom left |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15); |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| //manipulation for bottom- row 1 (row 1 right) |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 1); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| //bottom - row 1 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| |
| //eliminating old left for row 0,1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| signdwn1_16x8b = _mm_slli_si128(src_temp1_16x8b, 8); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //for the next iteration signup0_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) for next |
| |
| //storing right of row 1 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| //for storing right of row 1 |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| |
| src_top_16x8b = src_temp1_16x8b; |
| //storing right of row 0 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //the next top already in src_top_16x8b |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| cmp_lt0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 2; |
| pu1_src_left_str += 2; |
| } |
| ht_rem = ht & 0x1; |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| |
| //manipulation for row 0 -bottom |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 14); |
| //bottom left |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 15); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| //for row 0 right to put into left store |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); |
| edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8); |
| //left store manipulation 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1); |
| //filling the left boundary value |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 15); |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| src_top_16x8b = src_temp0_16x8b; |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_left_cpy++; |
| pu1_src_left_str++; |
| } |
| { //for bottom right |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 1); |
| src_temp0_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 15); |
| _mm_storel_epi64((__m128i *)(pu1_src_left_str), left_store_16x8b); |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| } |
| _mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 8; |
| |
| pu1_left_tmp = pu1_src_left_cpy2; |
| pu1_src_left_cpy2 = pu1_src_left_str2; |
| pu1_src_left_str2 = pu1_left_tmp; |
| |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| pu1_src_left_str = pu1_src_left_str2; |
| |
| } |
| pu1_src_org[wd - 1] = u1_pos_wd_0_tmp; |
| pu1_src_org[(ht_tmp - 1) * src_strd] = u1_pos_0_ht_tmp; |
| pu1_src_left_cpy = (0 == pu1_avail[2]) ? (pu1_src_left_cpy - 1) : pu1_src_left_cpy; |
| pu1_src_left[0] = au1_src_left_tmp[0]; |
| for(row = 1; row < ht_tmp; row++) |
| { |
| pu1_src_left[row] = pu1_src_left_cpy[row]; |
| } |
| } |
| |
| } |
| |
| void ihevc_sao_edge_offset_class3_chroma_ssse3(UWORD8 *pu1_src, |
| WORD32 src_strd, |
| UWORD8 *pu1_src_left, |
| UWORD8 *pu1_src_top, |
| UWORD8 *pu1_src_top_left, |
| UWORD8 *pu1_src_top_right, |
| UWORD8 *pu1_src_bot_left, |
| UWORD8 *pu1_avail, |
| WORD8 *pi1_sao_offset_u, |
| WORD8 *pi1_sao_offset_v, |
| WORD32 wd, |
| WORD32 ht) |
| { |
| WORD32 row, col; |
| UWORD8 *pu1_src_top_cpy, *pu1_src_left_cpy, *pu1_src_left_cpy2; |
| UWORD8 *pu1_src_cpy, *pu1_src_org; |
| UWORD8 au1_src_left_tmp[2 * (MAX_CTB_SIZE + 8)]; |
| UWORD8 au1_mask[MAX_CTB_SIZE], *au1_mask_cpy; |
| WORD32 wd_rem; |
| UWORD8 u1_pos_wd_0_tmp_u, u1_pos_wd_0_tmp_v, u1_pos_0_ht_tmp_u, u1_pos_0_ht_tmp_v; |
| WORD32 ht_tmp; |
| WORD32 bit_depth; |
| UWORD8 u1_avail0, u1_avail1; |
| |
| __m128i src_top_16x8b, src_bottom_16x8b; |
| __m128i src_temp0_16x8b, src_temp1_16x8b; |
| __m128i signup0_16x8b, signdwn1_16x8b; |
| __m128i cmp_gt0_16x8b, cmp_lt0_16x8b; |
| __m128i edge0_16x8b, edge1_16x8b; |
| __m128i au1_mask8x16b; |
| __m128i edge_idx_8x16b, sao_offset_8x16b; |
| __m128i left_store_16x8b; |
| __m128i const0_16x8b, const2_16x8b; |
| __m128i chroma_offset_8x16b; |
| |
| ht_tmp = ht; |
| au1_mask8x16b = _mm_set1_epi8(0xff); |
| |
| |
| au1_src_left_tmp[0] = pu1_src[(wd - 2)]; |
| au1_src_left_tmp[1] = pu1_src[(wd - 1)]; |
| //manipulation for bottom left |
| for(row = 2; row < 2 * ht; row++) |
| { |
| au1_src_left_tmp[row] = pu1_src_left[row]; |
| } |
| au1_src_left_tmp[2 * ht] = pu1_src_bot_left[0]; |
| au1_src_left_tmp[2 * ht + 1] = pu1_src_bot_left[1]; |
| |
| pu1_src_top_left[0] = pu1_src_top[wd - 2]; |
| pu1_src_top_left[1] = pu1_src_top[wd - 1]; |
| //setting availability mask to ff size MAX_CTB_SIZE |
| for(col = 0; col < MAX_CTB_SIZE; col += 16) |
| _mm_storeu_si128((__m128i *)(au1_mask + col), au1_mask8x16b); |
| bit_depth = BIT_DEPTH_LUMA; |
| pu1_src_org = pu1_src; |
| pu1_src_top_cpy = pu1_src_top; |
| pu1_src_left_cpy2 = au1_src_left_tmp; |
| pu1_src_left_cpy = au1_src_left_tmp; |
| edge_idx_8x16b = _mm_loadl_epi64((__m128i *)gi1_table_edge_idx); |
| sao_offset_8x16b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_u); |
| const0_16x8b = _mm_loadl_epi64((__m128i *)pi1_sao_offset_v); |
| chroma_offset_8x16b = _mm_set1_epi16(0x0800); |
| /* If top-right is available, process separately */ |
| if(0 != pu1_avail[5]) |
| { |
| WORD32 edge_idx; |
| |
| /* U */ |
| edge_idx = 2 + SIGN(pu1_src[wd - 2] - pu1_src_top_right[0]) + |
| SIGN(pu1_src[wd - 2] - pu1_src[wd - 2 - 2 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_wd_0_tmp_u = CLIP3(pu1_src[wd - 2] + pi1_sao_offset_u[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_wd_0_tmp_u = pu1_src[wd - 2]; |
| } |
| |
| /* V */ |
| edge_idx = 2 + SIGN(pu1_src[wd - 1] - pu1_src_top_right[1]) + |
| SIGN(pu1_src[wd - 1] - pu1_src[wd - 1 - 2 + src_strd]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_wd_0_tmp_v = CLIP3(pu1_src[wd - 1] + pi1_sao_offset_v[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_wd_0_tmp_v = pu1_src[wd - 1]; |
| } |
| } |
| else |
| { |
| u1_pos_wd_0_tmp_u = pu1_src[wd - 2]; |
| u1_pos_wd_0_tmp_v = pu1_src[wd - 1]; |
| } |
| |
| /* If bottom-left is available, process separately */ |
| if(0 != pu1_avail[6]) |
| { |
| WORD32 edge_idx; |
| |
| /* U */ |
| edge_idx = 2 + SIGN(pu1_src[(ht - 1) * src_strd] - pu1_src[(ht - 1) * src_strd + 2 - src_strd]) + |
| SIGN(pu1_src[(ht - 1) * src_strd] - pu1_src_bot_left[0]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_0_ht_tmp_u = CLIP3(pu1_src[(ht - 1) * src_strd] + pi1_sao_offset_u[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_0_ht_tmp_u = pu1_src[(ht - 1) * src_strd]; |
| } |
| |
| /* V */ |
| edge_idx = 2 + SIGN(pu1_src[(ht - 1) * src_strd + 1] - pu1_src[(ht - 1) * src_strd + 1 + 2 - src_strd]) + |
| SIGN(pu1_src[(ht - 1) * src_strd + 1] - pu1_src_bot_left[1]); |
| |
| edge_idx = gi1_table_edge_idx[edge_idx]; |
| |
| if(0 != edge_idx) |
| { |
| u1_pos_0_ht_tmp_v = CLIP3(pu1_src[(ht - 1) * src_strd + 1] + pi1_sao_offset_v[edge_idx], 0, (1 << bit_depth) - 1); |
| } |
| else |
| { |
| u1_pos_0_ht_tmp_v = pu1_src[(ht - 1) * src_strd + 1]; |
| } |
| } |
| else |
| { |
| u1_pos_0_ht_tmp_u = pu1_src[(ht - 1) * src_strd]; |
| u1_pos_0_ht_tmp_v = pu1_src[(ht - 1) * src_strd + 1]; |
| } |
| |
| |
| |
| /* Update height and source pointers based on the availability flags */ |
| if(0 == pu1_avail[2]) |
| { |
| pu1_src_left_cpy2 += 2; |
| pu1_src_top_cpy = pu1_src; |
| pu1_src += src_strd; |
| ht--; |
| } |
| if(0 == pu1_avail[3]) |
| { |
| ht--; |
| } |
| |
| sao_offset_8x16b = _mm_unpacklo_epi64(sao_offset_8x16b, const0_16x8b); |
| const2_16x8b = _mm_set1_epi8(2); |
| const0_16x8b = _mm_setzero_si128(); |
| |
| |
| //availability mask creation |
| u1_avail0 = pu1_avail[0]; |
| u1_avail1 = pu1_avail[1]; |
| au1_mask[0] = u1_avail0; |
| au1_mask[1] = u1_avail0; |
| au1_mask[wd - 1] = u1_avail1; |
| au1_mask[wd - 2] = u1_avail1; |
| { |
| WORD32 ht_rem; |
| au1_mask_cpy = au1_mask; |
| for(col = wd; col >= 16; col -= 16) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col + 2)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| |
| //loading the mask |
| au1_mask8x16b = _mm_loadu_si128((__m128i *)au1_mask_cpy); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| |
| for(row = ht; row >= 2; row -= 2) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_left_cpy)); |
| //row = 1 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| //to insert left in row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| // row = 0 right |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2)); |
| |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14); |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(0-1) |
| //combining sign-left and sign_right |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| |
| //row1-row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, src_bottom_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| // row = 1 right |
| signdwn1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd + 2)); |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-0) |
| |
| //bottom - row1 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //for the next iteration bottom -row1 |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //to insert left in row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 10); |
| //manipulation for row 1 - bottom |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14); |
| |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //combining sign-left and sign_right |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signdwn1_16x8b); |
| |
| //eliminating old left for row 0 and row 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4); |
| //row1 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp1_16x8b, 14); |
| //row0 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| //copying the next top |
| src_top_16x8b = src_temp1_16x8b; |
| |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b); |
| |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpackhi_epi8(src_temp1_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge1_16x8b); |
| src_temp1_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp1_16x8b); |
| //store left boundary |
| _mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy + src_strd), src_temp1_16x8b); |
| |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 4; |
| } |
| ht_rem = ht & 0x1; |
| |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| //to insert left in row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14); |
| |
| //current row -next row |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and bottom and constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| //eliminating old left for row 0 and row 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| //row0 getting it right for left of next block |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| //copying the next top |
| src_top_16x8b = src_temp0_16x8b; |
| |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| |
| |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| //store left boundary |
| _mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b); |
| |
| _mm_storeu_si128((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_left_cpy += 2; |
| } |
| { //for bottom right |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| _mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b); |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| } |
| //for the top left of next part of the block |
| left_store_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_top_cpy + wd - col)); |
| //updating top flag |
| _mm_storeu_si128((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 16; |
| au1_mask_cpy += 16; |
| } |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| wd_rem = wd & 0xF; |
| if(wd_rem) |
| { |
| pu1_src_cpy = pu1_src; |
| src_top_16x8b = _mm_loadl_epi64((__m128i *)(pu1_src_top_cpy + wd - col + 2)); |
| //row = 0 |
| src_temp0_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy)); |
| au1_mask8x16b = _mm_loadl_epi64((__m128i *)au1_mask_cpy); //???? |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, src_top_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(src_top_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //preparing au1_mask |
| au1_mask8x16b = _mm_unpacklo_epi64(au1_mask8x16b, au1_mask8x16b); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| pu1_src_left_cpy = pu1_src_left_cpy2; |
| for(row = ht; row >= 4; row -= 4) |
| { |
| left_store_16x8b = _mm_loadu_si128((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| //manipulation for row 0 -row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| //row 1 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14); |
| //row 0 -row1 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulatiing for row 1 -row 0 |
| signdwn1_16x8b = _mm_srli_si128(src_temp0_16x8b, 2); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //row 1 -row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row1-row0 |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //manipulation for row 1 -row 2 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 10); |
| //row 2 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14); |
| //packing row 0 n row 1 |
| src_temp0_16x8b = _mm_unpacklo_epi64(src_temp0_16x8b, src_temp1_16x8b); |
| //row1 -row2 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| |
| //row 1 right |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2); |
| //row = 3 |
| src_top_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 3 * src_strd)); |
| |
| // row = 4 |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 4 * src_strd)); |
| |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| |
| //separating +ve and and -ve values.(2,1) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //row 2 right |
| signdwn1_16x8b = _mm_srli_si128(src_bottom_16x8b, 2); |
| //combining the appropriate sign change |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) |
| |
| //separating +ve and and -ve values.(3,2) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b); |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); //aligned left (2-1) |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulation for row 2 -row 3 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 8); |
| //row 3 left |
| signdwn1_16x8b = _mm_alignr_epi8(src_top_16x8b, signdwn1_16x8b, 14); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-2) |
| |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(3-2) ,(2-1) |
| |
| //separating +ve and and -ve values.(2,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| |
| //manipulation for row 3 -bottom |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 6); |
| //bottom left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-3) |
| |
| //separating +ve and and -ve values.(3,bottom) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_top_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_top_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_slli_si128(edge1_16x8b, 8); //aligned left (2-3) |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(3-bottom) |
| edge1_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge1_16x8b, 8); //(3-bottom),(2-3) |
| |
| |
| //eliminating old left for row 0,1,2,3 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 8); |
| //packing row 2 n row 3 |
| src_bottom_16x8b = _mm_unpacklo_epi64(src_bottom_16x8b, src_top_16x8b); |
| //row 3 right |
| signdwn1_16x8b = _mm_srli_si128(src_top_16x8b, 2); |
| //loading row 3 right into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_bottom_16x8b, 14); |
| //adding bottom and top values of row 2 and row 3 |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, signup0_16x8b); //(3,2) |
| //separating +ve and and -ve values.(botttom,3) |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| //to store right of row 2 |
| signdwn1_16x8b = _mm_slli_si128(src_bottom_16x8b, 8); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(bottom -3) for next iteration |
| |
| //storing right of row 2into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| //to store right of row 0 |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //storing right of row 1 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| //storing right of row 0 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, const2_16x8b); |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge1_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| edge1_16x8b = _mm_and_si128(edge1_16x8b, au1_mask8x16b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| edge1_16x8b = _mm_add_epi8(edge1_16x8b, chroma_offset_8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| edge1_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge1_16x8b); |
| |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_unpackhi_epi8(src_temp0_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_temp0_16x8b); |
| |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge1_16x8b); |
| cmp_gt0_16x8b = _mm_unpacklo_epi8(src_bottom_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge1_16x8b, signdwn1_16x8b); |
| src_bottom_16x8b = _mm_unpackhi_epi8(src_bottom_16x8b, const0_16x8b); |
| edge1_16x8b = _mm_unpackhi_epi8(edge1_16x8b, signdwn1_16x8b); |
| cmp_gt0_16x8b = _mm_add_epi16(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| src_bottom_16x8b = _mm_add_epi16(src_bottom_16x8b, edge1_16x8b); |
| src_bottom_16x8b = _mm_packus_epi16(cmp_gt0_16x8b, src_bottom_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| cmp_lt0_16x8b = _mm_srli_si128(src_bottom_16x8b, 8); |
| _mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| //row = 2 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 2 * src_strd), src_bottom_16x8b); |
| // row = 3 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + 3 * src_strd), cmp_lt0_16x8b); |
| |
| src_temp0_16x8b = src_temp1_16x8b; |
| signup0_16x8b = _mm_slli_si128(signup0_16x8b, 8); |
| pu1_src_cpy += (src_strd << 2); |
| pu1_src_left_cpy += 8; |
| } |
| ht_rem = ht & 0x2; |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_temp1_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| // row = 2 |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + 2 * src_strd)); |
| |
| //manipulation for row 0 -row 1 |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| //bottom left |
| signdwn1_16x8b = _mm_alignr_epi8(src_temp1_16x8b, signdwn1_16x8b, 14); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //manipulation for row 1 - row 0 |
| signdwn1_16x8b = _mm_srli_si128(src_temp0_16x8b, 2); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //row1-row0 |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign chang |
| edge1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| |
| //manipulation for row 1 -bottom |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 10); |
| //bottom left |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14); |
| |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); //aligned left (0-1) |
| signup0_16x8b = _mm_alignr_epi8(edge1_16x8b, signup0_16x8b, 8); //(1-0),(0-top) |
| //row1 -bottom |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp1_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp1_16x8b); |
| |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| signdwn1_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(1-2) |
| edge0_16x8b = _mm_alignr_epi8(signdwn1_16x8b, edge0_16x8b, 8); //(1-2),(0-1) |
| |
| //manipulation for bottom- row 1 (row 1 right) |
| signdwn1_16x8b = _mm_srli_si128(src_temp1_16x8b, 2); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); //(1,0) sign_up empty |
| //bottom - row 1 |
| cmp_gt0_16x8b = _mm_subs_epu8(src_bottom_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_bottom_16x8b); |
| |
| //eliminating old left for row 0,1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 4); |
| signdwn1_16x8b = _mm_slli_si128(src_temp1_16x8b, 8); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //for the next iteration signup0_16x8b |
| signup0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); //(2-1) for next |
| |
| //storing right of row 1 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| //for storing right of row 1 |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| |
| src_top_16x8b = src_temp1_16x8b; |
| //storing right of row 0 into left |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| //the next top already in src_top_16x8b |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp1_16x8b = _mm_unpacklo_epi8(src_temp1_16x8b, const0_16x8b); |
| edge0_16x8b = _mm_unpackhi_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp1_16x8b = _mm_add_epi16(src_temp1_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, src_temp1_16x8b); |
| |
| cmp_gt0_16x8b = _mm_srli_si128(src_temp0_16x8b, 8); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| // row = 1 |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy + src_strd), cmp_gt0_16x8b); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_cpy += (src_strd << 1); |
| pu1_src_left_cpy += 4; |
| } |
| ht_rem = ht & 0x1; |
| if(ht_rem) |
| { |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| //row = 1 load 8 pixel values from 7:0 pos. relative to cur. pos. |
| src_bottom_16x8b = _mm_loadu_si128((__m128i *)(pu1_src_cpy + src_strd)); |
| |
| |
| //manipulation for row 0 -bottom |
| signdwn1_16x8b = _mm_slli_si128(left_store_16x8b, 12); |
| //bottom left |
| signdwn1_16x8b = _mm_alignr_epi8(src_bottom_16x8b, signdwn1_16x8b, 14); |
| //separating +ve and and -ve values. |
| cmp_gt0_16x8b = _mm_subs_epu8(src_temp0_16x8b, signdwn1_16x8b); |
| cmp_lt0_16x8b = _mm_subs_epu8(signdwn1_16x8b, src_temp0_16x8b); |
| //creating mask 00 for +ve and -ve values and FF for zero. |
| cmp_gt0_16x8b = _mm_cmpeq_epi8(cmp_gt0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_cmpeq_epi8(cmp_lt0_16x8b, const0_16x8b); |
| //combining the appropriate sign change |
| edge0_16x8b = _mm_sub_epi8(cmp_gt0_16x8b, cmp_lt0_16x8b); |
| //adding top and down substraction |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, signup0_16x8b); |
| //for row 0 right to put into left store |
| signdwn1_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| //adding constant 2 |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, const2_16x8b); |
| edge0_16x8b = _mm_slli_si128(edge0_16x8b, 8); |
| edge0_16x8b = _mm_srli_si128(edge0_16x8b, 8); |
| //left store manipulation 1 |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| //filling the left boundary value |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, signdwn1_16x8b, 14); |
| src_top_16x8b = src_temp0_16x8b; |
| |
| //shuffle to get sao index |
| edge0_16x8b = _mm_shuffle_epi8(edge_idx_8x16b, edge0_16x8b); |
| //using availability mask |
| edge0_16x8b = _mm_and_si128(edge0_16x8b, au1_mask8x16b); |
| //adding chroma offset to access U and V |
| edge0_16x8b = _mm_add_epi8(edge0_16x8b, chroma_offset_8x16b); |
| //shuffle to get sao offset |
| edge0_16x8b = _mm_shuffle_epi8(sao_offset_8x16b, edge0_16x8b); |
| |
| //cnvert to 16 bit then add and then saturated pack |
| signdwn1_16x8b = _mm_cmpgt_epi8(const0_16x8b, edge0_16x8b); |
| src_temp0_16x8b = _mm_unpacklo_epi8(src_temp0_16x8b, const0_16x8b); |
| cmp_lt0_16x8b = _mm_unpacklo_epi8(edge0_16x8b, signdwn1_16x8b); |
| src_temp0_16x8b = _mm_add_epi16(src_temp0_16x8b, cmp_lt0_16x8b); |
| src_temp0_16x8b = _mm_packus_epi16(src_temp0_16x8b, const0_16x8b); |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b); |
| //row = 0 store 8 pixel values from 7:0 pos. relative to cur. pos. |
| _mm_storel_epi64((__m128i *)(pu1_src_cpy), src_temp0_16x8b); |
| pu1_src_cpy += (src_strd); |
| src_temp0_16x8b = src_bottom_16x8b; |
| pu1_src_left_cpy += 2; |
| } |
| { //for bottom right |
| left_store_16x8b = _mm_loadl_epi64((__m128i *)pu1_src_left_cpy); |
| left_store_16x8b = _mm_srli_si128(left_store_16x8b, 2); |
| src_temp0_16x8b = _mm_slli_si128(src_temp0_16x8b, 8); |
| left_store_16x8b = _mm_alignr_epi8(left_store_16x8b, src_temp0_16x8b, 14); |
| _mm_storel_epi64((__m128i *)(pu1_src_left_cpy), left_store_16x8b); |
| } |
| if(0 == pu1_avail[3]) |
| { |
| src_top_16x8b = src_bottom_16x8b; |
| } |
| |
| _mm_storel_epi64((__m128i *)(pu1_src_top + wd - col), src_top_16x8b); |
| pu1_src += 8; |
| } |
| pu1_src_org[wd - 2] = u1_pos_wd_0_tmp_u; |
| pu1_src_org[wd - 1] = u1_pos_wd_0_tmp_v; |
| pu1_src_org[(ht_tmp - 1) * src_strd] = u1_pos_0_ht_tmp_u; |
| pu1_src_org[(ht_tmp - 1) * src_strd + 1] = u1_pos_0_ht_tmp_v; |
| for(row = 0; row < 2 * ht_tmp; row++) |
| { |
| pu1_src_left[row] = au1_src_left_tmp[row]; |
| } |
| } |
| |
| } |
