| // Copyright 2020 Google LLC |
| // |
| // This source code is licensed under the BSD-style license found in the |
| // LICENSE file in the root directory of this source tree. |
| |
| $SSE_HEADER = {2: "emmintrin.h", 4: "smmintrin.h"}[SSE] |
| $assert CHANNEL_TILE % 8 == 0 |
| $assert CHANNEL_TILE >= 8 |
| $assert ROW_TILE >= 2 |
| $assert ACCUMULATORS >= 1 |
| $assert ROW_TILE >= ACCUMULATORS * 2 |
| $assert REQUANTIZATION == "FP32" |
| $ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" |
| #include <assert.h> |
| |
| #include <${SSE_HEADER}> |
| |
| #include <xnnpack/gavgpool.h> |
| |
| |
| $PARAMS_STRUCT = "fp32_sse4" if SSE >= 4 else "fp32_sse2" |
| $ISA = {2: "sse2", 4: "sse41"}[SSE] |
| void xnn_qs8_gavgpool_minmax_fp32_ukernel_${ROW_TILE}x__${ISA}_c${CHANNEL_TILE}${"" if ACCUMULATORS == 1 else "_acc%d" % ACCUMULATORS}( |
| size_t rows, |
| size_t channels, |
| const int8_t* input, |
| size_t input_stride, |
| const int8_t* zero, |
| int8_t* output, |
| const union xnn_qs8_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS |
| { |
| assert(rows != 0); |
| assert(rows <= ${ROW_TILE}); |
| assert(channels != 0); |
| |
| const int8_t* i0 = input; |
| $for M in range(1, ROW_TILE): |
| const int8_t* i${M} = (const int8_t*) ((uintptr_t) i${M-1} + input_stride); |
| $if M % 2 == 1: |
| if XNN_UNPREDICTABLE(rows < ${M+1}) { |
| i${M} = zero; |
| } |
| $else: |
| if XNN_UNPREDICTABLE(rows <= ${M}) { |
| i${M} = zero; |
| } |
| |
| const __m128i vinit_bias = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.init_bias); |
| const __m128 vscale = _mm_load_ps(params->${PARAMS_STRUCT}.scale); |
| const __m128 voutput_max_less_zero_point = _mm_load_ps(params->${PARAMS_STRUCT}.output_max_less_zero_point); |
| const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_zero_point); |
| const __m128i voutput_min = _mm_load_si128((const __m128i*) params->${PARAMS_STRUCT}.output_min); |
| for (; channels >= ${CHANNEL_TILE}; channels -= ${CHANNEL_TILE}) { |
| $for M in range(ROW_TILE): |
| $if SSE >= 4: |
| const __m128i vxi${M}x${ABC[0:8]} = _mm_cvtepi8_epi16(_mm_loadl_epi64((const __m128i*) i${M})); |
| $for C in range(8, CHANNEL_TILE, 8): |
| const __m128i vxi${M}x${ABC[C:C+8]} = _mm_cvtepi8_epi16(_mm_loadl_epi64((const __m128i*) (i${M} + ${C}))); |
| $else: |
| const __m128i vi${M}x${ABC[0:8]} = _mm_loadl_epi64((const __m128i*) i${M}); |
| $for C in range(8, CHANNEL_TILE, 8): |
| const __m128i vi${M}x${ABC[C:C+8]} = _mm_loadl_epi64((const __m128i*) (i${M} + ${C})); |
| i${M} += ${CHANNEL_TILE}; |
| |
| $if SSE < 4: |
| $for M in range(ROW_TILE): |
| $for C in range(0, CHANNEL_TILE, 8): |
| const __m128i vxi${M}x${ABC[C:C+8]} = _mm_srai_epi16(_mm_unpacklo_epi8(vi${M}x${ABC[C:C+8]}, vi${M}x${ABC[C:C+8]}), 8); |
| |
| $for A in range(ACCUMULATORS): |
| $for C in range(0, CHANNEL_TILE, 8): |
| __m128i vacc${A}x${ABC[C:C+8]} = _mm_add_epi16(vxi${A*2}x${ABC[C:C+8]}, vxi${A*2+1}x${ABC[C:C+8]}); |
| |
| $for M in range(ACCUMULATORS * 2, ROW_TILE): |
| $for C in range(0, CHANNEL_TILE, 8): |
| vacc${M % ACCUMULATORS}x${ABC[C:C+8]} = _mm_add_epi16(vacc${M % ACCUMULATORS}x${ABC[C:C+8]}, vxi${M}x${ABC[C:C+8]}); |
| |
| $if ACCUMULATORS > 1: |
| // Add up all accumulators to vacc0x${ABC[0:CHANNEL_TILE]} |
| $ACC_SLICE = 1 |
| $while ACC_SLICE < ACCUMULATORS: |
| $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): |
| $if A + ACC_SLICE < ACCUMULATORS: |
| $for C in range(0, CHANNEL_TILE, 8): |
| vacc${A}x${ABC[C:C+8]} = _mm_add_epi16(vacc${A}x${ABC[C:C+8]}, vacc${A + ACC_SLICE}x${ABC[C:C+8]}); |
| $ACC_SLICE *= 2 |
| |
| $for C in range(0, CHANNEL_TILE, 8): |
| $if SSE == 4: |
| __m128i vacc${ABC[C:C+4]} = _mm_add_epi32(vinit_bias, _mm_cvtepi16_epi32(vacc0x${ABC[C:C+8]})); |
| __m128i vacc${ABC[C+4:C+8]} = _mm_add_epi32(vinit_bias, _mm_srai_epi32(_mm_unpackhi_epi16(vacc0x${ABC[C:C+8]}, vacc0x${ABC[C:C+8]}), 16)); |
| $else: |
| const __m128i vsgnacc0x${ABC[C:C+8]} = _mm_cmpgt_epi16(_mm_setzero_si128(), vacc0x${ABC[C:C+8]}); |
| __m128i vacc${ABC[C:C+4]} = _mm_add_epi32(vinit_bias, _mm_unpacklo_epi16(vacc0x${ABC[C:C+8]}, vsgnacc0x${ABC[C:C+8]})); |
| __m128i vacc${ABC[C+4:C+8]} = _mm_add_epi32(vinit_bias, _mm_unpackhi_epi16(vacc0x${ABC[C:C+8]}, vsgnacc0x${ABC[C:C+8]})); |
| |
| $for C in range(0, CHANNEL_TILE, 4): |
| __m128 vfpacc${ABC[C:C+4]} = _mm_cvtepi32_ps(vacc${ABC[C:C+4]}); |
| |
| $for C in range(0, CHANNEL_TILE, 4): |
| vfpacc${ABC[C:C+4]} = _mm_mul_ps(vfpacc${ABC[C:C+4]}, vscale); |
| |
| $for C in range(0, CHANNEL_TILE, 4): |
| vfpacc${ABC[C:C+4]} = _mm_min_ps(vfpacc${ABC[C:C+4]}, voutput_max_less_zero_point); |
| |
| $for C in range(0, CHANNEL_TILE, 4): |
| vacc${ABC[C:C+4]} = _mm_cvtps_epi32(vfpacc${ABC[C:C+4]}); |
| |
| $for C in range(0, CHANNEL_TILE, 8): |
| __m128i vout${ABC[C:C+8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point); |
| |
| $if SSE < 4: |
| $for C in range(0, CHANNEL_TILE, 8): |
| vout${ABC[C:C+8]} = _mm_max_epi16(vout${ABC[C:C+8]}, voutput_min); |
| |
| $for C in range(0, CHANNEL_TILE, 16): |
| $if C + 8 < CHANNEL_TILE: |
| __m128i vout${ABC[C:C+16]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C+8:C+16]}); |
| $else: |
| __m128i vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_packs_epi16(vout${ABC[C:C+8]}, vout${ABC[C:C+8]}); |
| |
| $if SSE == 4: |
| $for C in range(0, CHANNEL_TILE, 16): |
| $if C + 8 < CHANNEL_TILE: |
| vout${ABC[C:C+16]} = _mm_max_epi8(vout${ABC[C:C+16]}, voutput_min); |
| $else: |
| vout${ABC[C:C+8]}${ABC[C:C+8]} = _mm_max_epi8(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_min); |
| |
| $if CHANNEL_TILE > 8: |
| _mm_storeu_si128((__m128i*) output, vout${ABC[0:16]}); |
| $else: |
| _mm_storel_epi64((__m128i*) output, vout${ABC[0:8]}${ABC[0:8]}); |
| $for C in range(16, CHANNEL_TILE, 16): |
| $if C + 8 < CHANNEL_TILE: |
| _mm_storeu_si128((__m128i*) (output + ${C}), vout${ABC[C:C+16]}); |
| $else: |
| _mm_storel_epi64((__m128i*) (output + ${C}), vout${ABC[C:C+8]}${ABC[C:C+8]}); |
| output += ${CHANNEL_TILE}; |
| } |
| if XNN_UNLIKELY(channels != 0) { |
| ${"do " if CHANNEL_TILE > 8 else ""}{ |
| $for M in range(ROW_TILE): |
| $if SSE == 4: |
| const __m128i vxi${M}x${ABC[0:8]} = _mm_cvtepi8_epi16(_mm_loadl_epi64((const __m128i*) i${M})); |
| $else: |
| const __m128i vi${M}x${ABC[0:8]} = _mm_loadl_epi64((const __m128i*) i${M}); |
| i${M} += 8; |
| |
| $if SSE < 4: |
| $for M in range(ROW_TILE): |
| const __m128i vxi${M}x${ABC[0:8]} = _mm_srai_epi16(_mm_unpacklo_epi8(vi${M}x${ABC[0:8]}, vi${M}x${ABC[0:8]}), 8); |
| |
| $for A in range(ACCUMULATORS): |
| __m128i vacc${A}x${ABC[0:8]} = _mm_add_epi16(vxi${A*2}x${ABC[0:8]}, vxi${A*2+1}x${ABC[0:8]}); |
| |
| $for M in range(ACCUMULATORS * 2, ROW_TILE): |
| vacc${M % ACCUMULATORS}x${ABC[0:8]} = _mm_add_epi16(vacc${M % ACCUMULATORS}x${ABC[0:8]}, vxi${M}x${ABC[0:8]}); |
| |
| $if ACCUMULATORS > 1: |
| // Add up all accumulators to vacc0x${ABC[0:8]} |
| $ACC_SLICE = 1 |
| $while ACC_SLICE < ACCUMULATORS: |
| $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): |
| $if A + ACC_SLICE < ACCUMULATORS: |
| vacc${A}x${ABC[0:8]} = _mm_add_epi16(vacc${A}x${ABC[0:8]}, vacc${A + ACC_SLICE}x${ABC[0:8]}); |
| $ACC_SLICE *= 2 |
| |
| $if SSE == 4: |
| __m128i vacc${ABC[0:4]} = _mm_add_epi32(_mm_cvtepi16_epi32(vacc0x${ABC[0:8]}), vinit_bias); |
| __m128i vacc${ABC[4:8]} = _mm_add_epi32(_mm_srai_epi32(_mm_unpackhi_epi16(vacc0x${ABC[0:8]}, vacc0x${ABC[0:8]}), 16), vinit_bias); |
| $else: |
| const __m128i vsgnacc0x${ABC[0:8]} = _mm_cmpgt_epi16(_mm_setzero_si128(), vacc0x${ABC[0:8]}); |
| __m128i vacc${ABC[0:4]} = _mm_add_epi32(_mm_unpacklo_epi16(vacc0x${ABC[0:8]}, vsgnacc0x${ABC[0:8]}), vinit_bias); |
| __m128i vacc${ABC[4:8]} = _mm_add_epi32(_mm_unpackhi_epi16(vacc0x${ABC[0:8]}, vsgnacc0x${ABC[0:8]}), vinit_bias); |
| |
| __m128 vfpacc${ABC[0:4]} = _mm_cvtepi32_ps(vacc${ABC[0:4]}); |
| __m128 vfpacc${ABC[4:8]} = _mm_cvtepi32_ps(vacc${ABC[4:8]}); |
| |
| vfpacc${ABC[0:4]} = _mm_mul_ps(vfpacc${ABC[0:4]}, vscale); |
| vfpacc${ABC[4:8]} = _mm_mul_ps(vfpacc${ABC[4:8]}, vscale); |
| |
| vfpacc${ABC[0:4]} = _mm_min_ps(vfpacc${ABC[0:4]}, voutput_max_less_zero_point); |
| vfpacc${ABC[4:8]} = _mm_min_ps(vfpacc${ABC[4:8]}, voutput_max_less_zero_point); |
| |
| vacc${ABC[0:4]} = _mm_cvtps_epi32(vfpacc${ABC[0:4]}); |
| vacc${ABC[4:8]} = _mm_cvtps_epi32(vfpacc${ABC[4:8]}); |
| |
| __m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[4:8]}), voutput_zero_point); |
| $if SSE < 4: |
| vout${ABC[0:8]} = _mm_max_epi16(vout${ABC[0:8]}, voutput_min); |
| |
| __m128i vout${ABC[0:8]}${ABC[0:8]} = _mm_packs_epi16(vout${ABC[0:8]}, vout${ABC[0:8]}); |
| $if SSE == 4: |
| vout${ABC[0:8]}${ABC[0:8]} = _mm_max_epi8(vout${ABC[0:8]}${ABC[0:8]}, voutput_min); |
| |
| $if CHANNEL_TILE > 8: |
| if XNN_LIKELY(channels >= 8) { |
| _mm_storel_epi64((__m128i*) output, vout${ABC[0:8]}${ABC[0:8]}); |
| output += 8; |
| channels -= 8; |
| } else { |
| $if SSE == 4: |
| if (channels & 4) { |
| *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}); |
| vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi64(vout${ABC[0:8]}${ABC[0:8]}, 32); |
| output += 4; |
| } |
| if (channels & 2) { |
| *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout${ABC[0:8]}${ABC[0:8]}, 0); |
| vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi32(vout${ABC[0:8]}${ABC[0:8]}, 16); |
| output += 2; |
| } |
| if (channels & 1) { |
| *output = (int8_t) _mm_extract_epi8(vout${ABC[0:8]}${ABC[0:8]}, 0); |
| output += 1; |
| } |
| $else: |
| if (channels & 4) { |
| *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}); |
| vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi64(vout${ABC[0:8]}${ABC[0:8]}, 32); |
| output += 4; |
| } |
| uint32_t vout${ABC[0:4]} = (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}); |
| if (channels & 2) { |
| *((uint16_t*) output) = (uint16_t) vout${ABC[0:4]}; |
| vout${ABC[0:4]} >>= 16; |
| output += 2; |
| } |
| if (channels & 1) { |
| *output = (int8_t) vout${ABC[0:4]}; |
| output += 1; |
| } |
| channels = 0; |
| } |
| $else: |
| $if SSE == 4: |
| if (channels & 4) { |
| *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}); |
| vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi64(vout${ABC[0:8]}${ABC[0:8]}, 32); |
| output += 4; |
| } |
| if (channels & 2) { |
| *((uint16_t*) output) = (uint16_t) _mm_extract_epi16(vout${ABC[0:8]}${ABC[0:8]}, 0); |
| vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi32(vout${ABC[0:8]}${ABC[0:8]}, 16); |
| output += 2; |
| } |
| if (channels & 1) { |
| *output = (int8_t) _mm_extract_epi8(vout${ABC[0:8]}${ABC[0:8]}, 0); |
| } |
| $else: |
| if (channels & 4) { |
| *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}); |
| vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi64(vout${ABC[0:8]}${ABC[0:8]}, 32); |
| output += 4; |
| } |
| uint32_t vout${ABC[0:4]} = (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}); |
| if (channels & 2) { |
| *((uint16_t*) output) = (uint16_t) vout${ABC[0:4]}; |
| vout${ABC[0:4]} >>= 16; |
| output += 2; |
| } |
| if (channels & 1) { |
| *output = (int8_t) vout${ABC[0:4]}; |
| } |
| }${" while (channels != 0);" if CHANNEL_TILE > 8 else ""} |
| } |
| } |