NEON 4x16 QC8 GEMM and IGEMM assembly microkernels for Cortex A53
PiperOrigin-RevId: 382358109
diff --git a/BUILD.bazel b/BUILD.bazel
index 8c5b3ba..d4c5515 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -4336,6 +4336,10 @@
"src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-aarch64-neon-mlal-padal.S",
"src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S",
"src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S",
+ "src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S",
+ "src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S",
+ "src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S",
+ "src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S",
"src/qs8-gemm/gen/4x16c4-minmax-fp32-aarch64-neondot-cortex-a55.S",
"src/qs8-gemm/gen/4x16c4-minmax-fp32-aarch64-neondot-ld32.S",
"src/qs8-gemm/gen/4x16c4-minmax-fp32-aarch64-neondot-ld64.S",
@@ -4355,6 +4359,10 @@
"src/qs8-igemm/gen/2x8c16-minmax-gemmlowp-aarch64-neon-mlal-padal.S",
"src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S",
"src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S",
+ "src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S",
+ "src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S",
+ "src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S",
+ "src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S",
"src/qs8-igemm/gen/4x16c4-minmax-fp32-aarch64-neondot-cortex-a55.S",
"src/qs8-igemm/gen/4x16c4-minmax-fp32-aarch64-neondot-ld64.S",
"src/qs8-igemm/gen/4x16c4-minmax-fp32-aarch64-neondot-ld128.S",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index cbfcc91..2b560da 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3560,6 +3560,10 @@
src/qs8-gemm/gen/2x8c16-minmax-gemmlowp-aarch64-neon-mlal-padal.S
src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+ src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
+ src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+ src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
+ src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
src/qs8-gemm/gen/4x16c4-minmax-fp32-aarch64-neondot-cortex-a55.S
src/qs8-gemm/gen/4x16c4-minmax-fp32-aarch64-neondot-ld32.S
src/qs8-gemm/gen/4x16c4-minmax-fp32-aarch64-neondot-ld64.S
@@ -3579,6 +3583,10 @@
src/qs8-igemm/gen/2x8c16-minmax-gemmlowp-aarch64-neon-mlal-padal.S
src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+ src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
+ src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+ src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
+ src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
src/qs8-igemm/gen/4x16c4-minmax-fp32-aarch64-neondot-cortex-a55.S
src/qs8-igemm/gen/4x16c4-minmax-fp32-aarch64-neondot-ld64.S
src/qs8-igemm/gen/4x16c4-minmax-fp32-aarch64-neondot-ld128.S
diff --git a/scripts/generate-qs8-gemm.sh b/scripts/generate-qs8-gemm.sh
index 5504be0..2a9469d 100755
--- a/scripts/generate-qs8-gemm.sh
+++ b/scripts/generate-qs8-gemm.sh
@@ -208,8 +208,15 @@
tools/xngen src/qs8-gemm/1x8c8-aarch64-neon-mlal-padal-cortex-a53.S.in -D PREFETCH=1 -o src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-aarch64-neon-mlal-padal-prfm-cortex-a53.S
tools/xngen src/qs8-gemm/2x8c8-aarch64-neon-mlal-padal-cortex-a53.S.in -D PREFETCH=0 -o src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-aarch64-neon-mlal-padal-cortex-a53.S
tools/xngen src/qs8-gemm/2x8c8-aarch64-neon-mlal-padal-cortex-a53.S.in -D PREFETCH=1 -o src/qs8-gemm/gen/2x8c8-minmax-gemmlowp-aarch64-neon-mlal-padal-prfm-cortex-a53.S
-tools/xngen src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=0 -o src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
-tools/xngen src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=1 -o src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+
+tools/xngen src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=0 -D REQUANTIZATION=GEMMLOWP -D CHANNELWISE=0 -o src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
+tools/xngen src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=1 -D REQUANTIZATION=GEMMLOWP -D CHANNELWISE=0 -o src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+
+tools/xngen src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=0 -D REQUANTIZATION=FP32 -D CHANNELWISE=0 -o src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
+tools/xngen src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=1 -D REQUANTIZATION=FP32 -D CHANNELWISE=0 -o src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+
+tools/xngen src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=0 -D REQUANTIZATION=FP32 -D CHANNELWISE=1 -o src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
+tools/xngen src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=1 -D REQUANTIZATION=FP32 -D CHANNELWISE=1 -o src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
# Cortex A75 micro-kernel
tools/xngen src/qs8-gemm/1x8c8-aarch64-neon-mlal-padal.S.in -D PREFETCH=0 -o src/qs8-gemm/gen/1x8c8-minmax-gemmlowp-aarch64-neon-mlal-padal.S
diff --git a/scripts/generate-qs8-igemm.sh b/scripts/generate-qs8-igemm.sh
index 30f607d..02bff3b 100755
--- a/scripts/generate-qs8-igemm.sh
+++ b/scripts/generate-qs8-igemm.sh
@@ -205,8 +205,15 @@
tools/xngen src/qs8-igemm/1x8c8-aarch64-neon-mlal-padal-cortex-a53.S.in -D PREFETCH=1 -o src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-aarch64-neon-mlal-padal-prfm-cortex-a53.S
tools/xngen src/qs8-igemm/2x8c8-aarch64-neon-mlal-padal-cortex-a53.S.in -D PREFETCH=0 -o src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-aarch64-neon-mlal-padal-cortex-a53.S
tools/xngen src/qs8-igemm/2x8c8-aarch64-neon-mlal-padal-cortex-a53.S.in -D PREFETCH=1 -o src/qs8-igemm/gen/2x8c8-minmax-gemmlowp-aarch64-neon-mlal-padal-prfm-cortex-a53.S
-tools/xngen src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=0 -o src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
-tools/xngen src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=1 -o src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+
+tools/xngen src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=0 -D REQUANTIZATION=GEMMLOWP -D CHANNELWISE=0 -o src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
+tools/xngen src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=1 -D REQUANTIZATION=GEMMLOWP -D CHANNELWISE=0 -o src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+
+tools/xngen src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=0 -D REQUANTIZATION=FP32 -D CHANNELWISE=0 -o src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
+tools/xngen src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=1 -D REQUANTIZATION=FP32 -D CHANNELWISE=0 -o src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+
+tools/xngen src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=0 -D REQUANTIZATION=FP32 -D CHANNELWISE=1 -o src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
+tools/xngen src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in -D PREFETCH=1 -D REQUANTIZATION=FP32 -D CHANNELWISE=1 -o src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
# Cortex A75 micro-kernel
tools/xngen src/qs8-igemm/1x8c8-aarch64-neon-mlal-padal.S.in -D PREFETCH=0 -o src/qs8-igemm/gen/1x8c8-minmax-gemmlowp-aarch64-neon-mlal-padal.S
diff --git a/src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S b/src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
new file mode 100644
index 0000000..0cfcadc
--- /dev/null
+++ b/src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
@@ -0,0 +1,789 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+#include <xnnpack/assembly.h>
+
+# void xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53(
+# size_t mr, x0
+# size_t nc, x1
+# size_t kc, x2 / x0
+# const int8_t* restrict a, x3
+# size_t a_stride, x4
+# const void* restrict w, x5
+# int8_t* restrict c, x6
+# size_t cm_stride, x7
+# size_t cn_stride, [sp] -> x12
+# const union xnn_qs8_conv_minmax_params params) [sp + 8] -> x11
+
+# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
+
+# Register usage
+# A0 x3 v0
+# A1 x15 v1
+# A2 x13 v2
+# A3 x4 v3
+# B x5 v4 v5 v6
+# C0 x6 v16 v20 v24 v28
+# C1 x8 v17 v21 v25 v29
+# C2 x9 v18 v22 v26 v30
+# C3 x7 v19 v23 v27 v31
+# temp v7
+# unused v8 v9 v10 v11 v12 v13 v14 v15
+
+# x10 x17 a53 temp registers
+
+BEGIN_FUNCTION xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+
+ # Clamp A and C pointers
+ CMP x0, 2 // if mr < 2
+ ADD x15, x3, x4 // a1 = a0 + a_stride
+ ADD x8, x6, x7 // c1 = c0 + cm_stride
+ CSEL x15, x3, x15, LO // a1 = a0
+ CSEL x8, x6, x8, LO // c1 = c0
+
+ ADD x13, x15, x4 // a2 = a1 + a_stride
+ ADD x9, x8, x7 // c2 = c1 + cm_stride
+ // if mr <= 2
+ CSEL x13, x15, x13, LS // a2 = a1
+ CSEL x9, x8, x9, LS // c2 = c1
+
+ CMP x0, 4 // if mr < 4
+ ADD x4, x13, x4 // a3 = a2 + a_stride
+ ADD x7, x9, x7 // c3 = c2 + cm_stride
+ CSEL x4, x13, x4, LO // a3 = a2
+ CSEL x7, x9, x7, LO // c3 = c2
+
+ .p2align 3
+0:
+ # Load initial bias from w into accumulators
+ LDP q16, q20, [x5], 32
+ MOV v17.16b, v16.16b
+ MOV v18.16b, v16.16b
+ LDP q24, q28, [x5], 32
+ MOV v19.16b, v16.16b
+ MOV v21.16b, v20.16b
+ LDR x11, [sp, 8] // reload params
+ MOV v22.16b, v20.16b
+ MOV v23.16b, v20.16b
+ SUBS x0, x2, 8 // k = kc - 8
+ MOV v25.16b, v24.16b
+ MOV v26.16b, v24.16b
+ MOV v27.16b, v24.16b
+ MOV v29.16b, v28.16b
+ MOV v30.16b, v28.16b
+ MOV v31.16b, v28.16b
+ # Is there at least 8 bytes for epilogue?
+ B.LO 4f
+
+ # Prologue
+ LDR d0, [x3], 8
+ LDP d4, d6, [x5]
+ LDR d1, [x15], 8
+ LDR d2, [x13], 8
+ LDR d3, [x4], 8
+ SXTL v0.8h, v0.8b
+ LDR x17, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+
+ SUBS x0, x0, 8 // k = k - 8
+ # Is there at least 8 bytes for main loop?
+ B.LO 2f
+
+ # Main loop - 8 bytes of A
+ .p2align 3
+1:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x17, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x17, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x17, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x17, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x17, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ LDR x17, [x5, 112]
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ SXTL v4.8h, v4.8b
+ ADD x5, x5, 128
+
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ LDR x17, [x5]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SXTL v5.8h, v5.8b
+ LDR x10, [x3], 8
+
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ LDR d6, [x5, 8]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ LDR x17, [x13], 8
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ LDR d1, [x15], 8
+ INS v0.d[0], x10
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+ LDR d3, [x4], 8
+ INS v2.d[0], x17
+
+ SXTL v0.8h, v0.8b
+ SXTL v1.8h, v1.8b
+ LDR x17, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v2.8h, v2.8b
+ SUBS x0, x0, 8
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+ B.HS 1b
+
+ # Epilogue. Same as main loop but no preloads in final group
+
+ .p2align 3
+2:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x17, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x17, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x17, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x17, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x17, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR x17, [x5, 112]
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x17
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ ADD x5, x5, 128
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ AND x0, x2, 7 // kc remainder 0 to 7
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+
+ # Is there a remainder?- 1 to 7 bytes of A
+ CBNZ x0, 4f
+
+3:
+ # Load per channel scale values from weights
+ LDP q0, q1, [x5], 32
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ LDP q2, q3, [x5], 32
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v1.4s
+ FMUL v21.4s, v21.4s, v1.4s
+ FMUL v22.4s, v22.4s, v1.4s
+ FMUL v23.4s, v23.4s, v1.4s
+ FMUL v24.4s, v24.4s, v2.4s
+ FMUL v25.4s, v25.4s, v2.4s
+ FMUL v26.4s, v26.4s, v2.4s
+ FMUL v27.4s, v27.4s, v2.4s
+ FMUL v28.4s, v28.4s, v3.4s
+ FMUL v29.4s, v29.4s, v3.4s
+ FMUL v30.4s, v30.4s, v3.4s
+ FMUL v31.4s, v31.4s, v3.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
+
+ SQXTN v16.4h, v16.4s
+ SQXTN v17.4h, v17.4s
+ SQXTN v18.4h, v18.4s
+ SQXTN v19.4h, v19.4s
+ SQXTN v24.4h, v24.4s
+ SQXTN v25.4h, v25.4s
+ SQXTN v26.4h, v26.4s
+ SQXTN v27.4h, v27.4s
+ LD1R {v2.8h}, [x11], 2 // add bias
+
+ SQXTN2 v16.8h, v20.4s
+ SQXTN2 v17.8h, v21.4s
+ SQXTN2 v18.8h, v22.4s
+ SQXTN2 v19.8h, v23.4s
+ SQXTN2 v24.8h, v28.4s
+ SQXTN2 v25.8h, v29.4s
+ SQXTN2 v26.8h, v30.4s
+ SQXTN2 v27.8h, v31.4s
+
+ SQADD v16.8h, v16.8h, v2.8h
+ SQADD v17.8h, v17.8h, v2.8h
+ SQADD v18.8h, v18.8h, v2.8h
+ SQADD v19.8h, v19.8h, v2.8h
+ SQADD v24.8h, v24.8h, v2.8h
+ SQADD v25.8h, v25.8h, v2.8h
+ SQADD v26.8h, v26.8h, v2.8h
+ SQADD v27.8h, v27.8h, v2.8h
+ LD1R {v0.16b}, [x11], 1 // clamp min value
+
+ SQXTN v4.8b, v16.8h
+ SQXTN v5.8b, v17.8h
+ SQXTN v6.8b, v18.8h
+ SQXTN v7.8b, v19.8h
+ LD1R {v1.16b}, [x11] // clamp max value
+ SQXTN2 v4.16b, v24.8h
+ SQXTN2 v5.16b, v25.8h
+ SQXTN2 v6.16b, v26.8h
+ SQXTN2 v7.16b, v27.8h
+ LDR x12, [sp, 0] // cn_stride
+
+ SMAX v4.16b, v4.16b, v0.16b
+ SMAX v5.16b, v5.16b, v0.16b
+ SMAX v6.16b, v6.16b, v0.16b
+ SMAX v7.16b, v7.16b, v0.16b
+ SUBS x1, x1, 16
+ SMIN v4.16b, v4.16b, v1.16b
+ SMIN v5.16b, v5.16b, v1.16b
+ SMIN v6.16b, v6.16b, v1.16b
+ SMIN v7.16b, v7.16b, v1.16b
+ B.LO 5f
+
+ # Store full 4 x 16
+ ST1 {v4.16b}, [x6], x12
+ SUB x3, x3, x2 // a0 -= kc
+ ST1 {v5.16b}, [x8], x12
+ SUB x15, x15, x2 // a1 -= kc
+ ST1 {v6.16b}, [x9], x12
+ SUB x13, x13, x2 // a2 -= kc
+ ST1 {v7.16b}, [x7], x12
+ SUB x4, x4, x2 // a3 -= kc
+ B.NE 0b
+ RET
+
+ # Remainder- 1 to 7 bytes of A
+ .p2align 3
+4:
+ AND x0, x2, 7 // kc remainder 1 to 7
+
+ LD1 {v0.8b}, [x3], x0
+ LDP d4, d5, [x5], 16
+ LD1 {v1.8b}, [x15], x0
+ LD1 {v2.8b}, [x13], x0
+ LD1 {v3.8b}, [x4], x0
+ SXTL v0.8h, v0.8b
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v24.4s, v5.4h, v0.h[0]
+ SMLAL2 v28.4s, v5.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v25.4s, v5.4h, v1.h[0]
+ SMLAL2 v29.4s, v5.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v26.4s, v5.4h, v2.h[0]
+ SMLAL2 v30.4s, v5.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ SMLAL v27.4s, v5.4h, v3.h[0]
+ SMLAL2 v31.4s, v5.8h, v3.h[0]
+ CMP x0, 2
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[1]
+ SMLAL2 v20.4s, v4.8h, v0.h[1]
+ SMLAL v24.4s, v5.4h, v0.h[1]
+ SMLAL2 v28.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v4.4h, v1.h[1]
+ SMLAL2 v21.4s, v4.8h, v1.h[1]
+ SMLAL v25.4s, v5.4h, v1.h[1]
+ SMLAL2 v29.4s, v5.8h, v1.h[1]
+ SMLAL v18.4s, v4.4h, v2.h[1]
+ SMLAL2 v22.4s, v4.8h, v2.h[1]
+ SMLAL v26.4s, v5.4h, v2.h[1]
+ SMLAL2 v30.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v4.4h, v3.h[1]
+ SMLAL2 v23.4s, v4.8h, v3.h[1]
+ SMLAL v27.4s, v5.4h, v3.h[1]
+ SMLAL2 v31.4s, v5.8h, v3.h[1]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[2]
+ SMLAL2 v20.4s, v4.8h, v0.h[2]
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v17.4s, v4.4h, v1.h[2]
+ SMLAL2 v21.4s, v4.8h, v1.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SMLAL v18.4s, v4.4h, v2.h[2]
+ SMLAL2 v22.4s, v4.8h, v2.h[2]
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v19.4s, v4.4h, v3.h[2]
+ SMLAL2 v23.4s, v4.8h, v3.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ CMP x0, 4
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v24.4s, v5.4h, v0.h[3]
+ SMLAL2 v28.4s, v5.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SMLAL v25.4s, v5.4h, v1.h[3]
+ SMLAL2 v29.4s, v5.8h, v1.h[3]
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v26.4s, v5.4h, v2.h[3]
+ SMLAL2 v30.4s, v5.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ SMLAL v27.4s, v5.4h, v3.h[3]
+ SMLAL2 v31.4s, v5.8h, v3.h[3]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[4]
+ SMLAL2 v20.4s, v4.8h, v0.h[4]
+ SMLAL v24.4s, v5.4h, v0.h[4]
+ SMLAL2 v28.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v4.4h, v1.h[4]
+ SMLAL2 v21.4s, v4.8h, v1.h[4]
+ SMLAL v25.4s, v5.4h, v1.h[4]
+ SMLAL2 v29.4s, v5.8h, v1.h[4]
+ SMLAL v18.4s, v4.4h, v2.h[4]
+ SMLAL2 v22.4s, v4.8h, v2.h[4]
+ SMLAL v26.4s, v5.4h, v2.h[4]
+ SMLAL2 v30.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v4.4h, v3.h[4]
+ SMLAL2 v23.4s, v4.8h, v3.h[4]
+ SMLAL v27.4s, v5.4h, v3.h[4]
+ SMLAL2 v31.4s, v5.8h, v3.h[4]
+ CMP x0, 6
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[5]
+ SMLAL2 v20.4s, v4.8h, v0.h[5]
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v17.4s, v4.4h, v1.h[5]
+ SMLAL2 v21.4s, v4.8h, v1.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SMLAL v18.4s, v4.4h, v2.h[5]
+ SMLAL2 v22.4s, v4.8h, v2.h[5]
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v19.4s, v4.4h, v3.h[5]
+ SMLAL2 v23.4s, v4.8h, v3.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v24.4s, v5.4h, v0.h[6]
+ SMLAL2 v28.4s, v5.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v25.4s, v5.4h, v1.h[6]
+ SMLAL2 v29.4s, v5.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v26.4s, v5.4h, v2.h[6]
+ SMLAL2 v30.4s, v5.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ SMLAL v27.4s, v5.4h, v3.h[6]
+ SMLAL2 v31.4s, v5.8h, v3.h[6]
+ B 3b
+
+ # Store odd width
+ .p2align 3
+5:
+ TBZ x1, 3, 6f
+ STR d4, [x6], 8
+ DUP d4, v4.d[1]
+ STR d5, [x8], 8
+ DUP d5, v5.d[1]
+ STR d6, [x9], 8
+ DUP d6, v6.d[1]
+ STR d7, [x7], 8
+ DUP d7, v7.d[1]
+6:
+ TBZ x1, 2, 7f
+ STR s4, [x6], 4
+ DUP s4, v4.s[1]
+ STR s5, [x8], 4
+ DUP s5, v5.s[1]
+ STR s6, [x9], 4
+ DUP s6, v6.s[1]
+ STR s7, [x7], 4
+ DUP s7, v7.s[1]
+7:
+ TBZ x1, 1, 8f
+ ST1 {v4.h}[0], [x6], 2
+ DUP h4, v4.h[1]
+ ST1 {v5.h}[0], [x8], 2
+ DUP h5, v5.h[1]
+ ST1 {v6.h}[0], [x9], 2
+ DUP h6, v6.h[1]
+ ST1 {v7.h}[0], [x7], 2
+ DUP h7, v7.h[1]
+8:
+ TBZ x1, 0, 9f
+ ST1 {v4.b}[0], [x6]
+ ST1 {v5.b}[0], [x8]
+ ST1 {v6.b}[0], [x9]
+ ST1 {v7.b}[0], [x7]
+9:
+ RET
+
+END_FUNCTION xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+
+#ifdef __ELF__
+.section ".note.GNU-stack","",%progbits
+#endif
diff --git a/src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S b/src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
new file mode 100644
index 0000000..1b591ae
--- /dev/null
+++ b/src/qc8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
@@ -0,0 +1,795 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+#include <xnnpack/assembly.h>
+
+# void xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53(
+# size_t mr, x0
+# size_t nc, x1
+# size_t kc, x2 / x0
+# const int8_t* restrict a, x3
+# size_t a_stride, x4
+# const void* restrict w, x5
+# int8_t* restrict c, x6
+# size_t cm_stride, x7
+# size_t cn_stride, [sp] -> x12
+# const union xnn_qs8_conv_minmax_params params) [sp + 8] -> x11
+
+# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
+
+# Register usage
+# A0 x3 v0
+# A1 x15 v1
+# A2 x13 v2
+# A3 x4 v3
+# B x5 v4 v5 v6
+# C0 x6 v16 v20 v24 v28
+# C1 x8 v17 v21 v25 v29
+# C2 x9 v18 v22 v26 v30
+# C3 x7 v19 v23 v27 v31
+# temp v7
+# unused v8 v9 v10 v11 v12 v13 v14 v15
+
+# x10 x17 a53 temp registers
+
+BEGIN_FUNCTION xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+
+ # Clamp A and C pointers
+ CMP x0, 2 // if mr < 2
+ ADD x15, x3, x4 // a1 = a0 + a_stride
+ ADD x8, x6, x7 // c1 = c0 + cm_stride
+ CSEL x15, x3, x15, LO // a1 = a0
+ CSEL x8, x6, x8, LO // c1 = c0
+
+ ADD x13, x15, x4 // a2 = a1 + a_stride
+ ADD x9, x8, x7 // c2 = c1 + cm_stride
+ // if mr <= 2
+ CSEL x13, x15, x13, LS // a2 = a1
+ CSEL x9, x8, x9, LS // c2 = c1
+
+ CMP x0, 4 // if mr < 4
+ ADD x4, x13, x4 // a3 = a2 + a_stride
+ ADD x7, x9, x7 // c3 = c2 + cm_stride
+ CSEL x4, x13, x4, LO // a3 = a2
+ CSEL x7, x9, x7, LO // c3 = c2
+
+ .p2align 3
+0:
+ # Load initial bias from w into accumulators
+ LDP q16, q20, [x5], 32
+ MOV v17.16b, v16.16b
+ MOV v18.16b, v16.16b
+ LDP q24, q28, [x5], 32
+ MOV v19.16b, v16.16b
+ MOV v21.16b, v20.16b
+ LDR x11, [sp, 8] // reload params
+ MOV v22.16b, v20.16b
+ MOV v23.16b, v20.16b
+ SUBS x0, x2, 8 // k = kc - 8
+ MOV v25.16b, v24.16b
+ MOV v26.16b, v24.16b
+ MOV v27.16b, v24.16b
+ MOV v29.16b, v28.16b
+ MOV v30.16b, v28.16b
+ MOV v31.16b, v28.16b
+ # Is there at least 8 bytes for epilogue?
+ B.LO 4f
+
+ # Prologue
+ LDR d0, [x3], 8
+ LDP d4, d6, [x5]
+ LDR d1, [x15], 8
+ LDR d2, [x13], 8
+ LDR d3, [x4], 8
+ SXTL v0.8h, v0.8b
+ LDR x17, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+
+ SUBS x0, x0, 8 // k = k - 8
+ # Is there at least 8 bytes for main loop?
+ B.LO 2f
+
+ # Main loop - 8 bytes of A
+ .p2align 3
+1:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ PRFM PLDL1KEEP, [x3, 128]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ PRFM PLDL1KEEP, [x15, 128]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ PRFM PLDL1KEEP, [x13, 128]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ PRFM PLDL1KEEP, [x4, 128]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ PRFM PLDL1KEEP, [x5, 448]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ PRFM PLDL1KEEP, [x5, 512]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x17, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x17, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x17, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x17, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x17, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ LDR x17, [x5, 112]
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ SXTL v4.8h, v4.8b
+ ADD x5, x5, 128
+
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ LDR x17, [x5]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SXTL v5.8h, v5.8b
+ LDR x10, [x3], 8
+
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ LDR d6, [x5, 8]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ LDR x17, [x13], 8
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ LDR d1, [x15], 8
+ INS v0.d[0], x10
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+ LDR d3, [x4], 8
+ INS v2.d[0], x17
+
+ SXTL v0.8h, v0.8b
+ SXTL v1.8h, v1.8b
+ LDR x17, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v2.8h, v2.8b
+ SUBS x0, x0, 8
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+ B.HS 1b
+
+ # Epilogue. Same as main loop but no preloads in final group
+
+ .p2align 3
+2:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x17, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x17, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x17, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x17, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x17, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR x17, [x5, 112]
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x17
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ ADD x5, x5, 128
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ AND x0, x2, 7 // kc remainder 0 to 7
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+
+ # Is there a remainder?- 1 to 7 bytes of A
+ CBNZ x0, 4f
+
+3:
+ # Load per channel scale values from weights
+ LDP q0, q1, [x5], 32
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ LDP q2, q3, [x5], 32
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v1.4s
+ FMUL v21.4s, v21.4s, v1.4s
+ FMUL v22.4s, v22.4s, v1.4s
+ FMUL v23.4s, v23.4s, v1.4s
+ FMUL v24.4s, v24.4s, v2.4s
+ FMUL v25.4s, v25.4s, v2.4s
+ FMUL v26.4s, v26.4s, v2.4s
+ FMUL v27.4s, v27.4s, v2.4s
+ FMUL v28.4s, v28.4s, v3.4s
+ FMUL v29.4s, v29.4s, v3.4s
+ FMUL v30.4s, v30.4s, v3.4s
+ FMUL v31.4s, v31.4s, v3.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
+
+ SQXTN v16.4h, v16.4s
+ SQXTN v17.4h, v17.4s
+ SQXTN v18.4h, v18.4s
+ SQXTN v19.4h, v19.4s
+ SQXTN v24.4h, v24.4s
+ SQXTN v25.4h, v25.4s
+ SQXTN v26.4h, v26.4s
+ SQXTN v27.4h, v27.4s
+ LD1R {v2.8h}, [x11], 2 // add bias
+
+ SQXTN2 v16.8h, v20.4s
+ SQXTN2 v17.8h, v21.4s
+ SQXTN2 v18.8h, v22.4s
+ SQXTN2 v19.8h, v23.4s
+ SQXTN2 v24.8h, v28.4s
+ SQXTN2 v25.8h, v29.4s
+ SQXTN2 v26.8h, v30.4s
+ SQXTN2 v27.8h, v31.4s
+
+ SQADD v16.8h, v16.8h, v2.8h
+ SQADD v17.8h, v17.8h, v2.8h
+ SQADD v18.8h, v18.8h, v2.8h
+ SQADD v19.8h, v19.8h, v2.8h
+ SQADD v24.8h, v24.8h, v2.8h
+ SQADD v25.8h, v25.8h, v2.8h
+ SQADD v26.8h, v26.8h, v2.8h
+ SQADD v27.8h, v27.8h, v2.8h
+ LD1R {v0.16b}, [x11], 1 // clamp min value
+
+ SQXTN v4.8b, v16.8h
+ SQXTN v5.8b, v17.8h
+ SQXTN v6.8b, v18.8h
+ SQXTN v7.8b, v19.8h
+ LD1R {v1.16b}, [x11] // clamp max value
+ SQXTN2 v4.16b, v24.8h
+ SQXTN2 v5.16b, v25.8h
+ SQXTN2 v6.16b, v26.8h
+ SQXTN2 v7.16b, v27.8h
+ LDR x12, [sp, 0] // cn_stride
+
+ SMAX v4.16b, v4.16b, v0.16b
+ SMAX v5.16b, v5.16b, v0.16b
+ SMAX v6.16b, v6.16b, v0.16b
+ SMAX v7.16b, v7.16b, v0.16b
+ SUBS x1, x1, 16
+ SMIN v4.16b, v4.16b, v1.16b
+ SMIN v5.16b, v5.16b, v1.16b
+ SMIN v6.16b, v6.16b, v1.16b
+ SMIN v7.16b, v7.16b, v1.16b
+ B.LO 5f
+
+ # Store full 4 x 16
+ ST1 {v4.16b}, [x6], x12
+ SUB x3, x3, x2 // a0 -= kc
+ ST1 {v5.16b}, [x8], x12
+ SUB x15, x15, x2 // a1 -= kc
+ ST1 {v6.16b}, [x9], x12
+ SUB x13, x13, x2 // a2 -= kc
+ ST1 {v7.16b}, [x7], x12
+ SUB x4, x4, x2 // a3 -= kc
+ B.NE 0b
+ RET
+
+ # Remainder- 1 to 7 bytes of A
+ .p2align 3
+4:
+ AND x0, x2, 7 // kc remainder 1 to 7
+
+ LD1 {v0.8b}, [x3], x0
+ LDP d4, d5, [x5], 16
+ LD1 {v1.8b}, [x15], x0
+ LD1 {v2.8b}, [x13], x0
+ LD1 {v3.8b}, [x4], x0
+ SXTL v0.8h, v0.8b
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v24.4s, v5.4h, v0.h[0]
+ SMLAL2 v28.4s, v5.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v25.4s, v5.4h, v1.h[0]
+ SMLAL2 v29.4s, v5.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v26.4s, v5.4h, v2.h[0]
+ SMLAL2 v30.4s, v5.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ SMLAL v27.4s, v5.4h, v3.h[0]
+ SMLAL2 v31.4s, v5.8h, v3.h[0]
+ CMP x0, 2
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[1]
+ SMLAL2 v20.4s, v4.8h, v0.h[1]
+ SMLAL v24.4s, v5.4h, v0.h[1]
+ SMLAL2 v28.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v4.4h, v1.h[1]
+ SMLAL2 v21.4s, v4.8h, v1.h[1]
+ SMLAL v25.4s, v5.4h, v1.h[1]
+ SMLAL2 v29.4s, v5.8h, v1.h[1]
+ SMLAL v18.4s, v4.4h, v2.h[1]
+ SMLAL2 v22.4s, v4.8h, v2.h[1]
+ SMLAL v26.4s, v5.4h, v2.h[1]
+ SMLAL2 v30.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v4.4h, v3.h[1]
+ SMLAL2 v23.4s, v4.8h, v3.h[1]
+ SMLAL v27.4s, v5.4h, v3.h[1]
+ SMLAL2 v31.4s, v5.8h, v3.h[1]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[2]
+ SMLAL2 v20.4s, v4.8h, v0.h[2]
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v17.4s, v4.4h, v1.h[2]
+ SMLAL2 v21.4s, v4.8h, v1.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SMLAL v18.4s, v4.4h, v2.h[2]
+ SMLAL2 v22.4s, v4.8h, v2.h[2]
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v19.4s, v4.4h, v3.h[2]
+ SMLAL2 v23.4s, v4.8h, v3.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ CMP x0, 4
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v24.4s, v5.4h, v0.h[3]
+ SMLAL2 v28.4s, v5.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SMLAL v25.4s, v5.4h, v1.h[3]
+ SMLAL2 v29.4s, v5.8h, v1.h[3]
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v26.4s, v5.4h, v2.h[3]
+ SMLAL2 v30.4s, v5.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ SMLAL v27.4s, v5.4h, v3.h[3]
+ SMLAL2 v31.4s, v5.8h, v3.h[3]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[4]
+ SMLAL2 v20.4s, v4.8h, v0.h[4]
+ SMLAL v24.4s, v5.4h, v0.h[4]
+ SMLAL2 v28.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v4.4h, v1.h[4]
+ SMLAL2 v21.4s, v4.8h, v1.h[4]
+ SMLAL v25.4s, v5.4h, v1.h[4]
+ SMLAL2 v29.4s, v5.8h, v1.h[4]
+ SMLAL v18.4s, v4.4h, v2.h[4]
+ SMLAL2 v22.4s, v4.8h, v2.h[4]
+ SMLAL v26.4s, v5.4h, v2.h[4]
+ SMLAL2 v30.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v4.4h, v3.h[4]
+ SMLAL2 v23.4s, v4.8h, v3.h[4]
+ SMLAL v27.4s, v5.4h, v3.h[4]
+ SMLAL2 v31.4s, v5.8h, v3.h[4]
+ CMP x0, 6
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[5]
+ SMLAL2 v20.4s, v4.8h, v0.h[5]
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v17.4s, v4.4h, v1.h[5]
+ SMLAL2 v21.4s, v4.8h, v1.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SMLAL v18.4s, v4.4h, v2.h[5]
+ SMLAL2 v22.4s, v4.8h, v2.h[5]
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v19.4s, v4.4h, v3.h[5]
+ SMLAL2 v23.4s, v4.8h, v3.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v24.4s, v5.4h, v0.h[6]
+ SMLAL2 v28.4s, v5.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v25.4s, v5.4h, v1.h[6]
+ SMLAL2 v29.4s, v5.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v26.4s, v5.4h, v2.h[6]
+ SMLAL2 v30.4s, v5.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ SMLAL v27.4s, v5.4h, v3.h[6]
+ SMLAL2 v31.4s, v5.8h, v3.h[6]
+ B 3b
+
+ # Store odd width
+ .p2align 3
+5:
+ TBZ x1, 3, 6f
+ STR d4, [x6], 8
+ DUP d4, v4.d[1]
+ STR d5, [x8], 8
+ DUP d5, v5.d[1]
+ STR d6, [x9], 8
+ DUP d6, v6.d[1]
+ STR d7, [x7], 8
+ DUP d7, v7.d[1]
+6:
+ TBZ x1, 2, 7f
+ STR s4, [x6], 4
+ DUP s4, v4.s[1]
+ STR s5, [x8], 4
+ DUP s5, v5.s[1]
+ STR s6, [x9], 4
+ DUP s6, v6.s[1]
+ STR s7, [x7], 4
+ DUP s7, v7.s[1]
+7:
+ TBZ x1, 1, 8f
+ ST1 {v4.h}[0], [x6], 2
+ DUP h4, v4.h[1]
+ ST1 {v5.h}[0], [x8], 2
+ DUP h5, v5.h[1]
+ ST1 {v6.h}[0], [x9], 2
+ DUP h6, v6.h[1]
+ ST1 {v7.h}[0], [x7], 2
+ DUP h7, v7.h[1]
+8:
+ TBZ x1, 0, 9f
+ ST1 {v4.b}[0], [x6]
+ ST1 {v5.b}[0], [x8]
+ ST1 {v6.b}[0], [x9]
+ ST1 {v7.b}[0], [x7]
+9:
+ RET
+
+END_FUNCTION xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+
+#ifdef __ELF__
+.section ".note.GNU-stack","",%progbits
+#endif
diff --git a/src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S b/src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
new file mode 100644
index 0000000..c35843f
--- /dev/null
+++ b/src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
@@ -0,0 +1,817 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+// Generator: tools/xngen
+//
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+#include <xnnpack/assembly.h>
+
+# void xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53(
+# size_t mr, x0
+# size_t nc, x1
+# size_t kc, x2 / x0
+# size_t ks, x3 / x9
+# const int8_t**restrict a, x4
+# const int8_t* restrict w, x5
+# int8_t* restrict c, x6
+# size_t cm_stride, x7
+# size_t cn_stride, [sp] -> x10
+# size_t a_offset, [sp + 8] -> x11
+# const float* zero, [sp + 16] -> x12
+# const xnn_qs8_conv_minmax_params params [sp + 24] -> (x8)
+
+# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
+
+# Register usage
+# A0 x13 v0
+# A1 x14 v1
+# A2 x15 v2
+# A3 x20 v3
+# B x5 v4 v5 v6
+# C0 x6 v16 v20 v24 v28
+# C1 x16 v17 v21 v25 v29
+# C2 x17 v18 v22 v26 v30
+# C3 x7 v19 v23 v27 v31
+# temp v7
+# unused v8 v9 v10 v11 v12 v13 v14 v15
+# x8, x21 temp for Cortex-A53 loads
+
+BEGIN_FUNCTION xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+
+ # Clamp C pointers
+ CMP x0, 2 // if mr < 2
+ LDP x10, x11, [sp] // Load cn_stride, a_offset
+ ADD x16, x6, x7 // c1 = c0 + cm_stride
+ CSEL x16, x6, x16, LO // c1 = c0
+
+ ADD x17, x16, x7 // c2 = c1 + cm_stride
+ LDP x12, x8, [sp, 16] // Load zero, params pointer
+ // if mr <= 2
+ CSEL x17, x16, x17, LS // c2 = c1
+
+ CMP x0, 4 // if mr < 4
+ STP x20, x21, [sp, -16]! // Save x20-x21 on stack
+ ADD x7, x17, x7 // c3 = c2 + cm_stride
+ CSEL x7, x17, x7, LO // c3 = c2
+
+ .p2align 3
+0:
+ # Load initial bias from w into accumulators
+ LDP q16, q20, [x5], 32
+ MOV v17.16b, v16.16b
+ MOV v18.16b, v16.16b
+ LDP q24, q28, [x5], 32
+ MOV v19.16b, v16.16b
+ MOV v21.16b, v20.16b
+ MOV v22.16b, v20.16b
+ MOV v23.16b, v20.16b
+ MOV v25.16b, v24.16b
+ MOV v26.16b, v24.16b
+ MOV v27.16b, v24.16b
+ MOV v29.16b, v28.16b
+ MOV v30.16b, v28.16b
+ MOV v31.16b, v28.16b
+ MOV x9, x3 // p = ks
+
+ .p2align 3
+1:
+ # Load next 4 A pointers
+ LDP x13, x14, [x4], 16
+ LDP x15, x20, [x4], 16
+
+ CMP x13, x12 // if a0 == zero
+ ADD x13, x13, x11 // a0 += a_offset
+ CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
+ CMP x14, x12 // if a1 == zero
+ ADD x14, x14, x11 // a1 += a_offset
+ CSEL x14, x12, x14, EQ // a1 = zero, else += a1 + a_offset
+ CMP x15, x12 // if a2 == zero
+ ADD x15, x15, x11 // a2 += a_offset
+ CSEL x15, x12, x15, EQ // a2 = zero, else += a2 + a_offset
+ CMP x20, x12 // if a3 == zero
+ ADD x20, x20, x11 // a3 += a_offset
+ CSEL x20, x12, x20, EQ // a3 = zero, else += a3 + a_offset
+
+ # Is there at least 8 bytes for epilogue?
+ SUBS x0, x2, 8 // k = kc - 8
+ B.LO 5f
+
+ # Prologue
+ LDR d0, [x13], 8
+ LDP d4, d6, [x5]
+ LDR d1, [x14], 8
+ LDR d2, [x15], 8
+ LDR d3, [x20], 8
+ SXTL v0.8h, v0.8b
+ LDR x8, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+
+ SUBS x0, x0, 8 // k = k - 8
+ # Is there at least 8 bytes for main loop?
+ B.LO 3f
+
+ # Main loop - 8 bytes of A
+ .p2align 3
+2:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x8, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x8, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x8, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x8, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x8, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ LDR x8, [x5, 112]
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ SXTL v4.8h, v4.8b
+ ADD x5, x5, 128
+
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ LDR x8, [x5]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SXTL v5.8h, v5.8b
+ LDR x21, [x13], 8
+
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ LDR d6, [x5, 8]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ LDR x8, [x15], 8
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ LDR d1, [x14], 8
+ INS v0.d[0], x21
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+ LDR d3, [x20], 8
+ INS v2.d[0], x8
+
+ SXTL v0.8h, v0.8b
+ SXTL v1.8h, v1.8b
+ LDR x8, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v2.8h, v2.8b
+ SUBS x0, x0, 8
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+ B.HS 2b
+
+ # Epilogue. Same as main loop but no preloads in final group
+
+ .p2align 3
+3:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x8, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x8, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x8, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x8, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x8, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR x8, [x5, 112]
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x8
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ ADD x5, x5, 128
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ AND x0, x2, 7 // kc remainder 0 to 7
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ LDR x8, [sp, 40] // reload params pointer
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+
+ # Is there a remainder?- 1 to 7 bytes of A
+ CBNZ x0, 5f
+
+4:
+ # ks loop
+ SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
+ B.HI 1b
+
+ # Load per channel scale values from weights
+ LDP q0, q1, [x5], 32
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ LDP q2, q3, [x5], 32
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v1.4s
+ FMUL v21.4s, v21.4s, v1.4s
+ FMUL v22.4s, v22.4s, v1.4s
+ FMUL v23.4s, v23.4s, v1.4s
+ FMUL v24.4s, v24.4s, v2.4s
+ FMUL v25.4s, v25.4s, v2.4s
+ FMUL v26.4s, v26.4s, v2.4s
+ FMUL v27.4s, v27.4s, v2.4s
+ FMUL v28.4s, v28.4s, v3.4s
+ FMUL v29.4s, v29.4s, v3.4s
+ FMUL v30.4s, v30.4s, v3.4s
+ FMUL v31.4s, v31.4s, v3.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
+
+ SQXTN v16.4h, v16.4s
+ SQXTN v17.4h, v17.4s
+ SQXTN v18.4h, v18.4s
+ SQXTN v19.4h, v19.4s
+ SQXTN v24.4h, v24.4s
+ SQXTN v25.4h, v25.4s
+ SQXTN v26.4h, v26.4s
+ SQXTN v27.4h, v27.4s
+ LD1R {v2.8h}, [x8], 2 // add bias
+
+ SQXTN2 v16.8h, v20.4s
+ SQXTN2 v17.8h, v21.4s
+ SQXTN2 v18.8h, v22.4s
+ SQXTN2 v19.8h, v23.4s
+ SQXTN2 v24.8h, v28.4s
+ SQXTN2 v25.8h, v29.4s
+ SQXTN2 v26.8h, v30.4s
+ SQXTN2 v27.8h, v31.4s
+
+ SQADD v16.8h, v16.8h, v2.8h
+ SQADD v17.8h, v17.8h, v2.8h
+ SQADD v18.8h, v18.8h, v2.8h
+ SQADD v19.8h, v19.8h, v2.8h
+ SQADD v24.8h, v24.8h, v2.8h
+ SQADD v25.8h, v25.8h, v2.8h
+ SQADD v26.8h, v26.8h, v2.8h
+ SQADD v27.8h, v27.8h, v2.8h
+ LD1R {v0.16b}, [x8], 1 // clamp min value
+
+ SQXTN v4.8b, v16.8h
+ SQXTN v5.8b, v17.8h
+ SQXTN v6.8b, v18.8h
+ SQXTN v7.8b, v19.8h
+ LD1R {v1.16b}, [x8] // clamp max value
+ SQXTN2 v4.16b, v24.8h
+ SQXTN2 v5.16b, v25.8h
+ SQXTN2 v6.16b, v26.8h
+ SQXTN2 v7.16b, v27.8h
+ SUB x8, x8, 3 // rewind params pointer
+
+ SMAX v4.16b, v4.16b, v0.16b
+ SMAX v5.16b, v5.16b, v0.16b
+ SMAX v6.16b, v6.16b, v0.16b
+ SMAX v7.16b, v7.16b, v0.16b
+ SUBS x1, x1, 16
+ SMIN v4.16b, v4.16b, v1.16b
+ SMIN v5.16b, v5.16b, v1.16b
+ SMIN v6.16b, v6.16b, v1.16b
+ SMIN v7.16b, v7.16b, v1.16b
+ B.LO 6f
+
+ # Store full 4 x 16
+ ST1 {v7.16b}, [x7], x10
+ ST1 {v6.16b}, [x17], x10
+ ST1 {v5.16b}, [x16], x10
+ ST1 {v4.16b}, [x6], x10
+
+ SUB x4, x4, x3 // a -= ks
+
+ # nc loop
+ B.HI 0b
+
+ # Restore x20-x21 from stack
+ LDP x20, x21, [sp], 16
+ RET
+
+ # Remainder- 1 to 7 bytes of A
+ .p2align 3
+5:
+ AND x0, x2, 7 // kc remainder 1 to 7
+
+ LD1 {v0.8b}, [x13], x0
+ LDP d4, d5, [x5], 16
+ LD1 {v1.8b}, [x14], x0
+ LD1 {v2.8b}, [x15], x0
+ LD1 {v3.8b}, [x20], x0
+ SXTL v0.8h, v0.8b
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v24.4s, v5.4h, v0.h[0]
+ SMLAL2 v28.4s, v5.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v25.4s, v5.4h, v1.h[0]
+ SMLAL2 v29.4s, v5.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v26.4s, v5.4h, v2.h[0]
+ SMLAL2 v30.4s, v5.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ SMLAL v27.4s, v5.4h, v3.h[0]
+ SMLAL2 v31.4s, v5.8h, v3.h[0]
+ CMP x0, 2
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[1]
+ SMLAL2 v20.4s, v4.8h, v0.h[1]
+ SMLAL v24.4s, v5.4h, v0.h[1]
+ SMLAL2 v28.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v4.4h, v1.h[1]
+ SMLAL2 v21.4s, v4.8h, v1.h[1]
+ SMLAL v25.4s, v5.4h, v1.h[1]
+ SMLAL2 v29.4s, v5.8h, v1.h[1]
+ SMLAL v18.4s, v4.4h, v2.h[1]
+ SMLAL2 v22.4s, v4.8h, v2.h[1]
+ SMLAL v26.4s, v5.4h, v2.h[1]
+ SMLAL2 v30.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v4.4h, v3.h[1]
+ SMLAL2 v23.4s, v4.8h, v3.h[1]
+ SMLAL v27.4s, v5.4h, v3.h[1]
+ SMLAL2 v31.4s, v5.8h, v3.h[1]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[2]
+ SMLAL2 v20.4s, v4.8h, v0.h[2]
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v17.4s, v4.4h, v1.h[2]
+ SMLAL2 v21.4s, v4.8h, v1.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SMLAL v18.4s, v4.4h, v2.h[2]
+ SMLAL2 v22.4s, v4.8h, v2.h[2]
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v19.4s, v4.4h, v3.h[2]
+ SMLAL2 v23.4s, v4.8h, v3.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ CMP x0, 4
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v24.4s, v5.4h, v0.h[3]
+ SMLAL2 v28.4s, v5.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SMLAL v25.4s, v5.4h, v1.h[3]
+ SMLAL2 v29.4s, v5.8h, v1.h[3]
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v26.4s, v5.4h, v2.h[3]
+ SMLAL2 v30.4s, v5.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ SMLAL v27.4s, v5.4h, v3.h[3]
+ SMLAL2 v31.4s, v5.8h, v3.h[3]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[4]
+ SMLAL2 v20.4s, v4.8h, v0.h[4]
+ SMLAL v24.4s, v5.4h, v0.h[4]
+ SMLAL2 v28.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v4.4h, v1.h[4]
+ SMLAL2 v21.4s, v4.8h, v1.h[4]
+ SMLAL v25.4s, v5.4h, v1.h[4]
+ SMLAL2 v29.4s, v5.8h, v1.h[4]
+ SMLAL v18.4s, v4.4h, v2.h[4]
+ SMLAL2 v22.4s, v4.8h, v2.h[4]
+ SMLAL v26.4s, v5.4h, v2.h[4]
+ SMLAL2 v30.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v4.4h, v3.h[4]
+ SMLAL2 v23.4s, v4.8h, v3.h[4]
+ SMLAL v27.4s, v5.4h, v3.h[4]
+ SMLAL2 v31.4s, v5.8h, v3.h[4]
+ CMP x0, 6
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[5]
+ SMLAL2 v20.4s, v4.8h, v0.h[5]
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v17.4s, v4.4h, v1.h[5]
+ SMLAL2 v21.4s, v4.8h, v1.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SMLAL v18.4s, v4.4h, v2.h[5]
+ SMLAL2 v22.4s, v4.8h, v2.h[5]
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v19.4s, v4.4h, v3.h[5]
+ SMLAL2 v23.4s, v4.8h, v3.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v24.4s, v5.4h, v0.h[6]
+ SMLAL2 v28.4s, v5.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v25.4s, v5.4h, v1.h[6]
+ SMLAL2 v29.4s, v5.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v26.4s, v5.4h, v2.h[6]
+ SMLAL2 v30.4s, v5.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ SMLAL v27.4s, v5.4h, v3.h[6]
+ SMLAL2 v31.4s, v5.8h, v3.h[6]
+ B 4b
+
+ # Store odd width
+ .p2align 3
+6:
+ TBZ x1, 3, 7f
+ STR d7, [x7], 8
+ DUP d7, v7.d[1]
+ STR d6, [x17], 8
+ DUP d6, v6.d[1]
+ STR d5, [x16], 8
+ DUP d5, v5.d[1]
+ STR d4, [x6], 8
+ DUP d4, v4.d[1]
+7:
+ TBZ x1, 2, 8f
+ STR s7, [x7], 4
+ DUP s7, v7.s[1]
+ STR s6, [x17], 4
+ DUP s6, v6.s[1]
+ STR s5, [x16], 4
+ DUP s5, v5.s[1]
+ STR s4, [x6], 4
+ DUP s4, v4.s[1]
+8:
+ TBZ x1, 1, 9f
+ ST1 {v7.h}[0], [x7], 2
+ DUP h7, v7.h[1]
+ ST1 {v6.h}[0], [x17], 2
+ DUP h6, v6.h[1]
+ ST1 {v5.h}[0], [x16], 2
+ DUP h5, v5.h[1]
+ ST1 {v4.h}[0], [x6], 2
+ DUP h4, v4.h[1]
+9:
+ TBZ x1, 0, 10f
+ ST1 {v7.b}[0], [x7]
+ ST1 {v6.b}[0], [x17]
+ ST1 {v5.b}[0], [x16]
+ ST1 {v4.b}[0], [x6]
+10:
+ # Restore x20-x21 from stack
+ LDP x20, x21, [sp], 16
+ RET
+
+END_FUNCTION xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+
+#ifdef __ELF__
+.section ".note.GNU-stack","",%progbits
+#endif
diff --git a/src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S b/src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
new file mode 100644
index 0000000..71ad459
--- /dev/null
+++ b/src/qc8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
@@ -0,0 +1,823 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+// Generator: tools/xngen
+//
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+#include <xnnpack/assembly.h>
+
+# void xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53(
+# size_t mr, x0
+# size_t nc, x1
+# size_t kc, x2 / x0
+# size_t ks, x3 / x9
+# const int8_t**restrict a, x4
+# const int8_t* restrict w, x5
+# int8_t* restrict c, x6
+# size_t cm_stride, x7
+# size_t cn_stride, [sp] -> x10
+# size_t a_offset, [sp + 8] -> x11
+# const float* zero, [sp + 16] -> x12
+# const xnn_qs8_conv_minmax_params params [sp + 24] -> (x8)
+
+# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
+
+# Register usage
+# A0 x13 v0
+# A1 x14 v1
+# A2 x15 v2
+# A3 x20 v3
+# B x5 v4 v5 v6
+# C0 x6 v16 v20 v24 v28
+# C1 x16 v17 v21 v25 v29
+# C2 x17 v18 v22 v26 v30
+# C3 x7 v19 v23 v27 v31
+# temp v7
+# unused v8 v9 v10 v11 v12 v13 v14 v15
+# x8, x21 temp for Cortex-A53 loads
+
+BEGIN_FUNCTION xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+
+ # Clamp C pointers
+ CMP x0, 2 // if mr < 2
+ LDP x10, x11, [sp] // Load cn_stride, a_offset
+ ADD x16, x6, x7 // c1 = c0 + cm_stride
+ CSEL x16, x6, x16, LO // c1 = c0
+
+ ADD x17, x16, x7 // c2 = c1 + cm_stride
+ LDP x12, x8, [sp, 16] // Load zero, params pointer
+ // if mr <= 2
+ CSEL x17, x16, x17, LS // c2 = c1
+
+ CMP x0, 4 // if mr < 4
+ STP x20, x21, [sp, -16]! // Save x20-x21 on stack
+ ADD x7, x17, x7 // c3 = c2 + cm_stride
+ CSEL x7, x17, x7, LO // c3 = c2
+
+ .p2align 3
+0:
+ # Load initial bias from w into accumulators
+ LDP q16, q20, [x5], 32
+ MOV v17.16b, v16.16b
+ MOV v18.16b, v16.16b
+ LDP q24, q28, [x5], 32
+ MOV v19.16b, v16.16b
+ MOV v21.16b, v20.16b
+ MOV v22.16b, v20.16b
+ MOV v23.16b, v20.16b
+ MOV v25.16b, v24.16b
+ MOV v26.16b, v24.16b
+ MOV v27.16b, v24.16b
+ MOV v29.16b, v28.16b
+ MOV v30.16b, v28.16b
+ MOV v31.16b, v28.16b
+ MOV x9, x3 // p = ks
+
+ .p2align 3
+1:
+ # Load next 4 A pointers
+ LDP x13, x14, [x4], 16
+ LDP x15, x20, [x4], 16
+
+ CMP x13, x12 // if a0 == zero
+ ADD x13, x13, x11 // a0 += a_offset
+ CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
+ CMP x14, x12 // if a1 == zero
+ ADD x14, x14, x11 // a1 += a_offset
+ CSEL x14, x12, x14, EQ // a1 = zero, else += a1 + a_offset
+ CMP x15, x12 // if a2 == zero
+ ADD x15, x15, x11 // a2 += a_offset
+ CSEL x15, x12, x15, EQ // a2 = zero, else += a2 + a_offset
+ CMP x20, x12 // if a3 == zero
+ ADD x20, x20, x11 // a3 += a_offset
+ CSEL x20, x12, x20, EQ // a3 = zero, else += a3 + a_offset
+
+ # Is there at least 8 bytes for epilogue?
+ SUBS x0, x2, 8 // k = kc - 8
+ B.LO 5f
+
+ # Prologue
+ LDR d0, [x13], 8
+ LDP d4, d6, [x5]
+ LDR d1, [x14], 8
+ LDR d2, [x15], 8
+ LDR d3, [x20], 8
+ SXTL v0.8h, v0.8b
+ LDR x8, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+
+ SUBS x0, x0, 8 // k = k - 8
+ # Is there at least 8 bytes for main loop?
+ B.LO 3f
+
+ # Main loop - 8 bytes of A
+ .p2align 3
+2:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ PRFM PLDL1KEEP, [x13, 128]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ PRFM PLDL1KEEP, [x14, 128]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ PRFM PLDL1KEEP, [x15, 128]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ PRFM PLDL1KEEP, [x20, 128]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ PRFM PLDL1KEEP, [x5, 448]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ PRFM PLDL1KEEP, [x5, 512]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x8, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x8, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x8, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x8, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x8, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ LDR x8, [x5, 112]
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ SXTL v4.8h, v4.8b
+ ADD x5, x5, 128
+
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ LDR x8, [x5]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SXTL v5.8h, v5.8b
+ LDR x21, [x13], 8
+
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ LDR d6, [x5, 8]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ LDR x8, [x15], 8
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ LDR d1, [x14], 8
+ INS v0.d[0], x21
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+ LDR d3, [x20], 8
+ INS v2.d[0], x8
+
+ SXTL v0.8h, v0.8b
+ SXTL v1.8h, v1.8b
+ LDR x8, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v2.8h, v2.8b
+ SUBS x0, x0, 8
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+ B.HS 2b
+
+ # Epilogue. Same as main loop but no preloads in final group
+
+ .p2align 3
+3:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x8, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x8, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x8, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x8, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x8, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR x8, [x5, 112]
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x8
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ ADD x5, x5, 128
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ AND x0, x2, 7 // kc remainder 0 to 7
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ LDR x8, [sp, 40] // reload params pointer
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+
+ # Is there a remainder?- 1 to 7 bytes of A
+ CBNZ x0, 5f
+
+4:
+ # ks loop
+ SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
+ B.HI 1b
+
+ # Load per channel scale values from weights
+ LDP q0, q1, [x5], 32
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ LDP q2, q3, [x5], 32
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v1.4s
+ FMUL v21.4s, v21.4s, v1.4s
+ FMUL v22.4s, v22.4s, v1.4s
+ FMUL v23.4s, v23.4s, v1.4s
+ FMUL v24.4s, v24.4s, v2.4s
+ FMUL v25.4s, v25.4s, v2.4s
+ FMUL v26.4s, v26.4s, v2.4s
+ FMUL v27.4s, v27.4s, v2.4s
+ FMUL v28.4s, v28.4s, v3.4s
+ FMUL v29.4s, v29.4s, v3.4s
+ FMUL v30.4s, v30.4s, v3.4s
+ FMUL v31.4s, v31.4s, v3.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
+
+ SQXTN v16.4h, v16.4s
+ SQXTN v17.4h, v17.4s
+ SQXTN v18.4h, v18.4s
+ SQXTN v19.4h, v19.4s
+ SQXTN v24.4h, v24.4s
+ SQXTN v25.4h, v25.4s
+ SQXTN v26.4h, v26.4s
+ SQXTN v27.4h, v27.4s
+ LD1R {v2.8h}, [x8], 2 // add bias
+
+ SQXTN2 v16.8h, v20.4s
+ SQXTN2 v17.8h, v21.4s
+ SQXTN2 v18.8h, v22.4s
+ SQXTN2 v19.8h, v23.4s
+ SQXTN2 v24.8h, v28.4s
+ SQXTN2 v25.8h, v29.4s
+ SQXTN2 v26.8h, v30.4s
+ SQXTN2 v27.8h, v31.4s
+
+ SQADD v16.8h, v16.8h, v2.8h
+ SQADD v17.8h, v17.8h, v2.8h
+ SQADD v18.8h, v18.8h, v2.8h
+ SQADD v19.8h, v19.8h, v2.8h
+ SQADD v24.8h, v24.8h, v2.8h
+ SQADD v25.8h, v25.8h, v2.8h
+ SQADD v26.8h, v26.8h, v2.8h
+ SQADD v27.8h, v27.8h, v2.8h
+ LD1R {v0.16b}, [x8], 1 // clamp min value
+
+ SQXTN v4.8b, v16.8h
+ SQXTN v5.8b, v17.8h
+ SQXTN v6.8b, v18.8h
+ SQXTN v7.8b, v19.8h
+ LD1R {v1.16b}, [x8] // clamp max value
+ SQXTN2 v4.16b, v24.8h
+ SQXTN2 v5.16b, v25.8h
+ SQXTN2 v6.16b, v26.8h
+ SQXTN2 v7.16b, v27.8h
+ SUB x8, x8, 3 // rewind params pointer
+
+ SMAX v4.16b, v4.16b, v0.16b
+ SMAX v5.16b, v5.16b, v0.16b
+ SMAX v6.16b, v6.16b, v0.16b
+ SMAX v7.16b, v7.16b, v0.16b
+ SUBS x1, x1, 16
+ SMIN v4.16b, v4.16b, v1.16b
+ SMIN v5.16b, v5.16b, v1.16b
+ SMIN v6.16b, v6.16b, v1.16b
+ SMIN v7.16b, v7.16b, v1.16b
+ B.LO 6f
+
+ # Store full 4 x 16
+ ST1 {v7.16b}, [x7], x10
+ ST1 {v6.16b}, [x17], x10
+ ST1 {v5.16b}, [x16], x10
+ ST1 {v4.16b}, [x6], x10
+
+ SUB x4, x4, x3 // a -= ks
+
+ # nc loop
+ B.HI 0b
+
+ # Restore x20-x21 from stack
+ LDP x20, x21, [sp], 16
+ RET
+
+ # Remainder- 1 to 7 bytes of A
+ .p2align 3
+5:
+ AND x0, x2, 7 // kc remainder 1 to 7
+
+ LD1 {v0.8b}, [x13], x0
+ LDP d4, d5, [x5], 16
+ LD1 {v1.8b}, [x14], x0
+ LD1 {v2.8b}, [x15], x0
+ LD1 {v3.8b}, [x20], x0
+ SXTL v0.8h, v0.8b
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v24.4s, v5.4h, v0.h[0]
+ SMLAL2 v28.4s, v5.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v25.4s, v5.4h, v1.h[0]
+ SMLAL2 v29.4s, v5.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v26.4s, v5.4h, v2.h[0]
+ SMLAL2 v30.4s, v5.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ SMLAL v27.4s, v5.4h, v3.h[0]
+ SMLAL2 v31.4s, v5.8h, v3.h[0]
+ CMP x0, 2
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[1]
+ SMLAL2 v20.4s, v4.8h, v0.h[1]
+ SMLAL v24.4s, v5.4h, v0.h[1]
+ SMLAL2 v28.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v4.4h, v1.h[1]
+ SMLAL2 v21.4s, v4.8h, v1.h[1]
+ SMLAL v25.4s, v5.4h, v1.h[1]
+ SMLAL2 v29.4s, v5.8h, v1.h[1]
+ SMLAL v18.4s, v4.4h, v2.h[1]
+ SMLAL2 v22.4s, v4.8h, v2.h[1]
+ SMLAL v26.4s, v5.4h, v2.h[1]
+ SMLAL2 v30.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v4.4h, v3.h[1]
+ SMLAL2 v23.4s, v4.8h, v3.h[1]
+ SMLAL v27.4s, v5.4h, v3.h[1]
+ SMLAL2 v31.4s, v5.8h, v3.h[1]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[2]
+ SMLAL2 v20.4s, v4.8h, v0.h[2]
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v17.4s, v4.4h, v1.h[2]
+ SMLAL2 v21.4s, v4.8h, v1.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SMLAL v18.4s, v4.4h, v2.h[2]
+ SMLAL2 v22.4s, v4.8h, v2.h[2]
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v19.4s, v4.4h, v3.h[2]
+ SMLAL2 v23.4s, v4.8h, v3.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ CMP x0, 4
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v24.4s, v5.4h, v0.h[3]
+ SMLAL2 v28.4s, v5.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SMLAL v25.4s, v5.4h, v1.h[3]
+ SMLAL2 v29.4s, v5.8h, v1.h[3]
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v26.4s, v5.4h, v2.h[3]
+ SMLAL2 v30.4s, v5.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ SMLAL v27.4s, v5.4h, v3.h[3]
+ SMLAL2 v31.4s, v5.8h, v3.h[3]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[4]
+ SMLAL2 v20.4s, v4.8h, v0.h[4]
+ SMLAL v24.4s, v5.4h, v0.h[4]
+ SMLAL2 v28.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v4.4h, v1.h[4]
+ SMLAL2 v21.4s, v4.8h, v1.h[4]
+ SMLAL v25.4s, v5.4h, v1.h[4]
+ SMLAL2 v29.4s, v5.8h, v1.h[4]
+ SMLAL v18.4s, v4.4h, v2.h[4]
+ SMLAL2 v22.4s, v4.8h, v2.h[4]
+ SMLAL v26.4s, v5.4h, v2.h[4]
+ SMLAL2 v30.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v4.4h, v3.h[4]
+ SMLAL2 v23.4s, v4.8h, v3.h[4]
+ SMLAL v27.4s, v5.4h, v3.h[4]
+ SMLAL2 v31.4s, v5.8h, v3.h[4]
+ CMP x0, 6
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[5]
+ SMLAL2 v20.4s, v4.8h, v0.h[5]
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v17.4s, v4.4h, v1.h[5]
+ SMLAL2 v21.4s, v4.8h, v1.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SMLAL v18.4s, v4.4h, v2.h[5]
+ SMLAL2 v22.4s, v4.8h, v2.h[5]
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v19.4s, v4.4h, v3.h[5]
+ SMLAL2 v23.4s, v4.8h, v3.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v24.4s, v5.4h, v0.h[6]
+ SMLAL2 v28.4s, v5.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v25.4s, v5.4h, v1.h[6]
+ SMLAL2 v29.4s, v5.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v26.4s, v5.4h, v2.h[6]
+ SMLAL2 v30.4s, v5.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ SMLAL v27.4s, v5.4h, v3.h[6]
+ SMLAL2 v31.4s, v5.8h, v3.h[6]
+ B 4b
+
+ # Store odd width
+ .p2align 3
+6:
+ TBZ x1, 3, 7f
+ STR d7, [x7], 8
+ DUP d7, v7.d[1]
+ STR d6, [x17], 8
+ DUP d6, v6.d[1]
+ STR d5, [x16], 8
+ DUP d5, v5.d[1]
+ STR d4, [x6], 8
+ DUP d4, v4.d[1]
+7:
+ TBZ x1, 2, 8f
+ STR s7, [x7], 4
+ DUP s7, v7.s[1]
+ STR s6, [x17], 4
+ DUP s6, v6.s[1]
+ STR s5, [x16], 4
+ DUP s5, v5.s[1]
+ STR s4, [x6], 4
+ DUP s4, v4.s[1]
+8:
+ TBZ x1, 1, 9f
+ ST1 {v7.h}[0], [x7], 2
+ DUP h7, v7.h[1]
+ ST1 {v6.h}[0], [x17], 2
+ DUP h6, v6.h[1]
+ ST1 {v5.h}[0], [x16], 2
+ DUP h5, v5.h[1]
+ ST1 {v4.h}[0], [x6], 2
+ DUP h4, v4.h[1]
+9:
+ TBZ x1, 0, 10f
+ ST1 {v7.b}[0], [x7]
+ ST1 {v6.b}[0], [x17]
+ ST1 {v5.b}[0], [x16]
+ ST1 {v4.b}[0], [x6]
+10:
+ # Restore x20-x21 from stack
+ LDP x20, x21, [sp], 16
+ RET
+
+END_FUNCTION xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+
+#ifdef __ELF__
+.section ".note.GNU-stack","",%progbits
+#endif
diff --git a/src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in b/src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
index a0e9e9a..6ebc143 100644
--- a/src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+++ b/src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
@@ -3,9 +3,14 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
+$assert not CHANNELWISE or REQUANTIZATION == "FP32"
+
#include <xnnpack/assembly.h>
-# void xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53(
+$DATATYPE = "qc8" if CHANNELWISE else "qs8"
+$CONV_PARAMS = "xnn_qs8_minmax_params" if CHANNELWISE else "xnn_qs8_conv_minmax_params"
+# void xnn_${DATATYPE}_gemm_minmax_${REQUANTIZATION.lower()}_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53(
# size_t mr, x0
# size_t nc, x1
# size_t kc, x2 / x0
@@ -34,7 +39,7 @@
# x10 x17 a53 temp registers
-BEGIN_FUNCTION xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53
+BEGIN_FUNCTION xnn_${DATATYPE}_gemm_minmax_${REQUANTIZATION.lower()}_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53
# Clamp A and C pointers
CMP x0, 2 // if mr < 2
@@ -64,7 +69,7 @@
LDP q24, q28, [x5], 32
MOV v19.16b, v16.16b
MOV v21.16b, v20.16b
- LDR x11, [sp, 8] // params
+ LDR x11, [sp, 8] // reload params
MOV v22.16b, v20.16b
MOV v23.16b, v20.16b
SUBS x0, x2, 8 // k = kc - 8
@@ -468,86 +473,163 @@
CBNZ x0, 4f
3:
- # Apply params - scale, shift, bias and clamp
- LD2R {v0.4s, v1.4s}, [x11], 8
- CMEQ v2.4s, v1.4s, 0
+ $if REQUANTIZATION == "GEMMLOWP":
+ # Apply params - scale, shift, bias and clamp
+ LD2R {v0.4s, v1.4s}, [x11], 8
+ CMEQ v2.4s, v1.4s, 0
- BIC v4.16b, v16.16b, v2.16b
- BIC v5.16b, v17.16b, v2.16b
- BIC v6.16b, v18.16b, v2.16b
- BIC v7.16b, v19.16b, v2.16b
+ BIC v4.16b, v16.16b, v2.16b
+ BIC v5.16b, v17.16b, v2.16b
+ BIC v6.16b, v18.16b, v2.16b
+ BIC v7.16b, v19.16b, v2.16b
- SQRDMULH v16.4s, v16.4s, v0.4s
- SQRDMULH v17.4s, v17.4s, v0.4s
- SQRDMULH v18.4s, v18.4s, v0.4s
- SQRDMULH v19.4s, v19.4s, v0.4s
+ SQRDMULH v16.4s, v16.4s, v0.4s
+ SQRDMULH v17.4s, v17.4s, v0.4s
+ SQRDMULH v18.4s, v18.4s, v0.4s
+ SQRDMULH v19.4s, v19.4s, v0.4s
- SSRA v16.4s, v4.4s, 31 // signed shift right accumulate
- SSRA v17.4s, v5.4s, 31
- SSRA v18.4s, v6.4s, 31
- SSRA v19.4s, v7.4s, 31
+ SSRA v16.4s, v4.4s, 31 // signed shift right accumulate
+ SSRA v17.4s, v5.4s, 31
+ SSRA v18.4s, v6.4s, 31
+ SSRA v19.4s, v7.4s, 31
- BIC v4.16b, v20.16b, v2.16b
- BIC v5.16b, v21.16b, v2.16b
- BIC v6.16b, v22.16b, v2.16b
- BIC v7.16b, v23.16b, v2.16b
+ BIC v4.16b, v20.16b, v2.16b
+ BIC v5.16b, v21.16b, v2.16b
+ BIC v6.16b, v22.16b, v2.16b
+ BIC v7.16b, v23.16b, v2.16b
- SQRDMULH v20.4s, v20.4s, v0.4s
- SQRDMULH v21.4s, v21.4s, v0.4s
- SQRDMULH v22.4s, v22.4s, v0.4s
- SQRDMULH v23.4s, v23.4s, v0.4s
+ SQRDMULH v20.4s, v20.4s, v0.4s
+ SQRDMULH v21.4s, v21.4s, v0.4s
+ SQRDMULH v22.4s, v22.4s, v0.4s
+ SQRDMULH v23.4s, v23.4s, v0.4s
- SSRA v20.4s, v4.4s, 31
- SSRA v21.4s, v5.4s, 31
- SSRA v22.4s, v6.4s, 31
- SSRA v23.4s, v7.4s, 31
+ SSRA v20.4s, v4.4s, 31
+ SSRA v21.4s, v5.4s, 31
+ SSRA v22.4s, v6.4s, 31
+ SSRA v23.4s, v7.4s, 31
- BIC v4.16b, v24.16b, v2.16b
- BIC v5.16b, v25.16b, v2.16b
- BIC v6.16b, v26.16b, v2.16b
- BIC v7.16b, v27.16b, v2.16b
+ BIC v4.16b, v24.16b, v2.16b
+ BIC v5.16b, v25.16b, v2.16b
+ BIC v6.16b, v26.16b, v2.16b
+ BIC v7.16b, v27.16b, v2.16b
- SQRDMULH v24.4s, v24.4s, v0.4s
- SQRDMULH v25.4s, v25.4s, v0.4s
- SQRDMULH v26.4s, v26.4s, v0.4s
- SQRDMULH v27.4s, v27.4s, v0.4s
+ SQRDMULH v24.4s, v24.4s, v0.4s
+ SQRDMULH v25.4s, v25.4s, v0.4s
+ SQRDMULH v26.4s, v26.4s, v0.4s
+ SQRDMULH v27.4s, v27.4s, v0.4s
- SSRA v24.4s, v4.4s, 31
- SSRA v25.4s, v5.4s, 31
- SSRA v26.4s, v6.4s, 31
- SSRA v27.4s, v7.4s, 31
+ SSRA v24.4s, v4.4s, 31
+ SSRA v25.4s, v5.4s, 31
+ SSRA v26.4s, v6.4s, 31
+ SSRA v27.4s, v7.4s, 31
- BIC v4.16b, v28.16b, v2.16b
- BIC v5.16b, v29.16b, v2.16b
- BIC v6.16b, v30.16b, v2.16b
- BIC v7.16b, v31.16b, v2.16b
+ BIC v4.16b, v28.16b, v2.16b
+ BIC v5.16b, v29.16b, v2.16b
+ BIC v6.16b, v30.16b, v2.16b
+ BIC v7.16b, v31.16b, v2.16b
- SQRDMULH v28.4s, v28.4s, v0.4s
- SQRDMULH v29.4s, v29.4s, v0.4s
- SQRDMULH v30.4s, v30.4s, v0.4s
- SQRDMULH v31.4s, v31.4s, v0.4s
+ SQRDMULH v28.4s, v28.4s, v0.4s
+ SQRDMULH v29.4s, v29.4s, v0.4s
+ SQRDMULH v30.4s, v30.4s, v0.4s
+ SQRDMULH v31.4s, v31.4s, v0.4s
- SSRA v28.4s, v4.4s, 31
- SSRA v29.4s, v5.4s, 31
- SSRA v30.4s, v6.4s, 31
- SSRA v31.4s, v7.4s, 31
+ SSRA v28.4s, v4.4s, 31
+ SSRA v29.4s, v5.4s, 31
+ SSRA v30.4s, v6.4s, 31
+ SSRA v31.4s, v7.4s, 31
- SRSHL v16.4s, v16.4s, v1.4s // signed rounding shift left
- SRSHL v17.4s, v17.4s, v1.4s
- SRSHL v18.4s, v18.4s, v1.4s
- SRSHL v19.4s, v19.4s, v1.4s
- SRSHL v20.4s, v20.4s, v1.4s
- SRSHL v21.4s, v21.4s, v1.4s
- SRSHL v22.4s, v22.4s, v1.4s
- SRSHL v23.4s, v23.4s, v1.4s
- SRSHL v24.4s, v24.4s, v1.4s
- SRSHL v25.4s, v25.4s, v1.4s
- SRSHL v26.4s, v26.4s, v1.4s
- SRSHL v27.4s, v27.4s, v1.4s
- SRSHL v28.4s, v28.4s, v1.4s
- SRSHL v29.4s, v29.4s, v1.4s
- SRSHL v30.4s, v30.4s, v1.4s
- SRSHL v31.4s, v31.4s, v1.4s
+ SRSHL v16.4s, v16.4s, v1.4s // signed rounding shift left
+ SRSHL v17.4s, v17.4s, v1.4s
+ SRSHL v18.4s, v18.4s, v1.4s
+ SRSHL v19.4s, v19.4s, v1.4s
+ SRSHL v20.4s, v20.4s, v1.4s
+ SRSHL v21.4s, v21.4s, v1.4s
+ SRSHL v22.4s, v22.4s, v1.4s
+ SRSHL v23.4s, v23.4s, v1.4s
+ SRSHL v24.4s, v24.4s, v1.4s
+ SRSHL v25.4s, v25.4s, v1.4s
+ SRSHL v26.4s, v26.4s, v1.4s
+ SRSHL v27.4s, v27.4s, v1.4s
+ SRSHL v28.4s, v28.4s, v1.4s
+ SRSHL v29.4s, v29.4s, v1.4s
+ SRSHL v30.4s, v30.4s, v1.4s
+ SRSHL v31.4s, v31.4s, v1.4s
+ $elif REQUANTIZATION == "FP32":
+ $if CHANNELWISE:
+ # Load per channel scale values from weights
+ LDP q0, q1, [x5], 32
+ $else:
+ # Apply params - scale, bias and clamp
+ LD1R {v0.4s}, [x11], 4
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ $if CHANNELWISE:
+ LDP q2, q3, [x5], 32
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v1.4s
+ FMUL v21.4s, v21.4s, v1.4s
+ FMUL v22.4s, v22.4s, v1.4s
+ FMUL v23.4s, v23.4s, v1.4s
+ FMUL v24.4s, v24.4s, v2.4s
+ FMUL v25.4s, v25.4s, v2.4s
+ FMUL v26.4s, v26.4s, v2.4s
+ FMUL v27.4s, v27.4s, v2.4s
+ FMUL v28.4s, v28.4s, v3.4s
+ FMUL v29.4s, v29.4s, v3.4s
+ FMUL v30.4s, v30.4s, v3.4s
+ FMUL v31.4s, v31.4s, v3.4s
+ $else:
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v0.4s
+ FMUL v21.4s, v21.4s, v0.4s
+ FMUL v22.4s, v22.4s, v0.4s
+ FMUL v23.4s, v23.4s, v0.4s
+ FMUL v24.4s, v24.4s, v0.4s
+ FMUL v25.4s, v25.4s, v0.4s
+ FMUL v26.4s, v26.4s, v0.4s
+ FMUL v27.4s, v27.4s, v0.4s
+ FMUL v28.4s, v28.4s, v0.4s
+ FMUL v29.4s, v29.4s, v0.4s
+ FMUL v30.4s, v30.4s, v0.4s
+ FMUL v31.4s, v31.4s, v0.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
SQXTN v16.4h, v16.4s
SQXTN v17.4h, v17.4s
@@ -816,7 +898,7 @@
9:
RET
-END_FUNCTION xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53
+END_FUNCTION xnn_${DATATYPE}_gemm_minmax_${REQUANTIZATION.lower()}_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53
#ifdef __ELF__
.section ".note.GNU-stack","",%progbits
diff --git a/src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S b/src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
new file mode 100644
index 0000000..eec0409
--- /dev/null
+++ b/src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
@@ -0,0 +1,788 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+#include <xnnpack/assembly.h>
+
+# void xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53(
+# size_t mr, x0
+# size_t nc, x1
+# size_t kc, x2 / x0
+# const int8_t* restrict a, x3
+# size_t a_stride, x4
+# const void* restrict w, x5
+# int8_t* restrict c, x6
+# size_t cm_stride, x7
+# size_t cn_stride, [sp] -> x12
+# const union xnn_qs8_conv_minmax_params params) [sp + 8] -> x11
+
+# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
+
+# Register usage
+# A0 x3 v0
+# A1 x15 v1
+# A2 x13 v2
+# A3 x4 v3
+# B x5 v4 v5 v6
+# C0 x6 v16 v20 v24 v28
+# C1 x8 v17 v21 v25 v29
+# C2 x9 v18 v22 v26 v30
+# C3 x7 v19 v23 v27 v31
+# temp v7
+# unused v8 v9 v10 v11 v12 v13 v14 v15
+
+# x10 x17 a53 temp registers
+
+BEGIN_FUNCTION xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+
+ # Clamp A and C pointers
+ CMP x0, 2 // if mr < 2
+ ADD x15, x3, x4 // a1 = a0 + a_stride
+ ADD x8, x6, x7 // c1 = c0 + cm_stride
+ CSEL x15, x3, x15, LO // a1 = a0
+ CSEL x8, x6, x8, LO // c1 = c0
+
+ ADD x13, x15, x4 // a2 = a1 + a_stride
+ ADD x9, x8, x7 // c2 = c1 + cm_stride
+ // if mr <= 2
+ CSEL x13, x15, x13, LS // a2 = a1
+ CSEL x9, x8, x9, LS // c2 = c1
+
+ CMP x0, 4 // if mr < 4
+ ADD x4, x13, x4 // a3 = a2 + a_stride
+ ADD x7, x9, x7 // c3 = c2 + cm_stride
+ CSEL x4, x13, x4, LO // a3 = a2
+ CSEL x7, x9, x7, LO // c3 = c2
+
+ .p2align 3
+0:
+ # Load initial bias from w into accumulators
+ LDP q16, q20, [x5], 32
+ MOV v17.16b, v16.16b
+ MOV v18.16b, v16.16b
+ LDP q24, q28, [x5], 32
+ MOV v19.16b, v16.16b
+ MOV v21.16b, v20.16b
+ LDR x11, [sp, 8] // reload params
+ MOV v22.16b, v20.16b
+ MOV v23.16b, v20.16b
+ SUBS x0, x2, 8 // k = kc - 8
+ MOV v25.16b, v24.16b
+ MOV v26.16b, v24.16b
+ MOV v27.16b, v24.16b
+ MOV v29.16b, v28.16b
+ MOV v30.16b, v28.16b
+ MOV v31.16b, v28.16b
+ # Is there at least 8 bytes for epilogue?
+ B.LO 4f
+
+ # Prologue
+ LDR d0, [x3], 8
+ LDP d4, d6, [x5]
+ LDR d1, [x15], 8
+ LDR d2, [x13], 8
+ LDR d3, [x4], 8
+ SXTL v0.8h, v0.8b
+ LDR x17, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+
+ SUBS x0, x0, 8 // k = k - 8
+ # Is there at least 8 bytes for main loop?
+ B.LO 2f
+
+ # Main loop - 8 bytes of A
+ .p2align 3
+1:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x17, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x17, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x17, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x17, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x17, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ LDR x17, [x5, 112]
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ SXTL v4.8h, v4.8b
+ ADD x5, x5, 128
+
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ LDR x17, [x5]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SXTL v5.8h, v5.8b
+ LDR x10, [x3], 8
+
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ LDR d6, [x5, 8]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ LDR x17, [x13], 8
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ LDR d1, [x15], 8
+ INS v0.d[0], x10
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+ LDR d3, [x4], 8
+ INS v2.d[0], x17
+
+ SXTL v0.8h, v0.8b
+ SXTL v1.8h, v1.8b
+ LDR x17, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v2.8h, v2.8b
+ SUBS x0, x0, 8
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+ B.HS 1b
+
+ # Epilogue. Same as main loop but no preloads in final group
+
+ .p2align 3
+2:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x17, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x17, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x17, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x17, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x17, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR x17, [x5, 112]
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x17
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ ADD x5, x5, 128
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ AND x0, x2, 7 // kc remainder 0 to 7
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+
+ # Is there a remainder?- 1 to 7 bytes of A
+ CBNZ x0, 4f
+
+3:
+ # Apply params - scale, bias and clamp
+ LD1R {v0.4s}, [x11], 4
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v0.4s
+ FMUL v21.4s, v21.4s, v0.4s
+ FMUL v22.4s, v22.4s, v0.4s
+ FMUL v23.4s, v23.4s, v0.4s
+ FMUL v24.4s, v24.4s, v0.4s
+ FMUL v25.4s, v25.4s, v0.4s
+ FMUL v26.4s, v26.4s, v0.4s
+ FMUL v27.4s, v27.4s, v0.4s
+ FMUL v28.4s, v28.4s, v0.4s
+ FMUL v29.4s, v29.4s, v0.4s
+ FMUL v30.4s, v30.4s, v0.4s
+ FMUL v31.4s, v31.4s, v0.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
+
+ SQXTN v16.4h, v16.4s
+ SQXTN v17.4h, v17.4s
+ SQXTN v18.4h, v18.4s
+ SQXTN v19.4h, v19.4s
+ SQXTN v24.4h, v24.4s
+ SQXTN v25.4h, v25.4s
+ SQXTN v26.4h, v26.4s
+ SQXTN v27.4h, v27.4s
+ LD1R {v2.8h}, [x11], 2 // add bias
+
+ SQXTN2 v16.8h, v20.4s
+ SQXTN2 v17.8h, v21.4s
+ SQXTN2 v18.8h, v22.4s
+ SQXTN2 v19.8h, v23.4s
+ SQXTN2 v24.8h, v28.4s
+ SQXTN2 v25.8h, v29.4s
+ SQXTN2 v26.8h, v30.4s
+ SQXTN2 v27.8h, v31.4s
+
+ SQADD v16.8h, v16.8h, v2.8h
+ SQADD v17.8h, v17.8h, v2.8h
+ SQADD v18.8h, v18.8h, v2.8h
+ SQADD v19.8h, v19.8h, v2.8h
+ SQADD v24.8h, v24.8h, v2.8h
+ SQADD v25.8h, v25.8h, v2.8h
+ SQADD v26.8h, v26.8h, v2.8h
+ SQADD v27.8h, v27.8h, v2.8h
+ LD1R {v0.16b}, [x11], 1 // clamp min value
+
+ SQXTN v4.8b, v16.8h
+ SQXTN v5.8b, v17.8h
+ SQXTN v6.8b, v18.8h
+ SQXTN v7.8b, v19.8h
+ LD1R {v1.16b}, [x11] // clamp max value
+ SQXTN2 v4.16b, v24.8h
+ SQXTN2 v5.16b, v25.8h
+ SQXTN2 v6.16b, v26.8h
+ SQXTN2 v7.16b, v27.8h
+ LDR x12, [sp, 0] // cn_stride
+
+ SMAX v4.16b, v4.16b, v0.16b
+ SMAX v5.16b, v5.16b, v0.16b
+ SMAX v6.16b, v6.16b, v0.16b
+ SMAX v7.16b, v7.16b, v0.16b
+ SUBS x1, x1, 16
+ SMIN v4.16b, v4.16b, v1.16b
+ SMIN v5.16b, v5.16b, v1.16b
+ SMIN v6.16b, v6.16b, v1.16b
+ SMIN v7.16b, v7.16b, v1.16b
+ B.LO 5f
+
+ # Store full 4 x 16
+ ST1 {v4.16b}, [x6], x12
+ SUB x3, x3, x2 // a0 -= kc
+ ST1 {v5.16b}, [x8], x12
+ SUB x15, x15, x2 // a1 -= kc
+ ST1 {v6.16b}, [x9], x12
+ SUB x13, x13, x2 // a2 -= kc
+ ST1 {v7.16b}, [x7], x12
+ SUB x4, x4, x2 // a3 -= kc
+ B.NE 0b
+ RET
+
+ # Remainder- 1 to 7 bytes of A
+ .p2align 3
+4:
+ AND x0, x2, 7 // kc remainder 1 to 7
+
+ LD1 {v0.8b}, [x3], x0
+ LDP d4, d5, [x5], 16
+ LD1 {v1.8b}, [x15], x0
+ LD1 {v2.8b}, [x13], x0
+ LD1 {v3.8b}, [x4], x0
+ SXTL v0.8h, v0.8b
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v24.4s, v5.4h, v0.h[0]
+ SMLAL2 v28.4s, v5.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v25.4s, v5.4h, v1.h[0]
+ SMLAL2 v29.4s, v5.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v26.4s, v5.4h, v2.h[0]
+ SMLAL2 v30.4s, v5.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ SMLAL v27.4s, v5.4h, v3.h[0]
+ SMLAL2 v31.4s, v5.8h, v3.h[0]
+ CMP x0, 2
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[1]
+ SMLAL2 v20.4s, v4.8h, v0.h[1]
+ SMLAL v24.4s, v5.4h, v0.h[1]
+ SMLAL2 v28.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v4.4h, v1.h[1]
+ SMLAL2 v21.4s, v4.8h, v1.h[1]
+ SMLAL v25.4s, v5.4h, v1.h[1]
+ SMLAL2 v29.4s, v5.8h, v1.h[1]
+ SMLAL v18.4s, v4.4h, v2.h[1]
+ SMLAL2 v22.4s, v4.8h, v2.h[1]
+ SMLAL v26.4s, v5.4h, v2.h[1]
+ SMLAL2 v30.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v4.4h, v3.h[1]
+ SMLAL2 v23.4s, v4.8h, v3.h[1]
+ SMLAL v27.4s, v5.4h, v3.h[1]
+ SMLAL2 v31.4s, v5.8h, v3.h[1]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[2]
+ SMLAL2 v20.4s, v4.8h, v0.h[2]
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v17.4s, v4.4h, v1.h[2]
+ SMLAL2 v21.4s, v4.8h, v1.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SMLAL v18.4s, v4.4h, v2.h[2]
+ SMLAL2 v22.4s, v4.8h, v2.h[2]
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v19.4s, v4.4h, v3.h[2]
+ SMLAL2 v23.4s, v4.8h, v3.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ CMP x0, 4
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v24.4s, v5.4h, v0.h[3]
+ SMLAL2 v28.4s, v5.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SMLAL v25.4s, v5.4h, v1.h[3]
+ SMLAL2 v29.4s, v5.8h, v1.h[3]
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v26.4s, v5.4h, v2.h[3]
+ SMLAL2 v30.4s, v5.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ SMLAL v27.4s, v5.4h, v3.h[3]
+ SMLAL2 v31.4s, v5.8h, v3.h[3]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[4]
+ SMLAL2 v20.4s, v4.8h, v0.h[4]
+ SMLAL v24.4s, v5.4h, v0.h[4]
+ SMLAL2 v28.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v4.4h, v1.h[4]
+ SMLAL2 v21.4s, v4.8h, v1.h[4]
+ SMLAL v25.4s, v5.4h, v1.h[4]
+ SMLAL2 v29.4s, v5.8h, v1.h[4]
+ SMLAL v18.4s, v4.4h, v2.h[4]
+ SMLAL2 v22.4s, v4.8h, v2.h[4]
+ SMLAL v26.4s, v5.4h, v2.h[4]
+ SMLAL2 v30.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v4.4h, v3.h[4]
+ SMLAL2 v23.4s, v4.8h, v3.h[4]
+ SMLAL v27.4s, v5.4h, v3.h[4]
+ SMLAL2 v31.4s, v5.8h, v3.h[4]
+ CMP x0, 6
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[5]
+ SMLAL2 v20.4s, v4.8h, v0.h[5]
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v17.4s, v4.4h, v1.h[5]
+ SMLAL2 v21.4s, v4.8h, v1.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SMLAL v18.4s, v4.4h, v2.h[5]
+ SMLAL2 v22.4s, v4.8h, v2.h[5]
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v19.4s, v4.4h, v3.h[5]
+ SMLAL2 v23.4s, v4.8h, v3.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v24.4s, v5.4h, v0.h[6]
+ SMLAL2 v28.4s, v5.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v25.4s, v5.4h, v1.h[6]
+ SMLAL2 v29.4s, v5.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v26.4s, v5.4h, v2.h[6]
+ SMLAL2 v30.4s, v5.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ SMLAL v27.4s, v5.4h, v3.h[6]
+ SMLAL2 v31.4s, v5.8h, v3.h[6]
+ B 3b
+
+ # Store odd width
+ .p2align 3
+5:
+ TBZ x1, 3, 6f
+ STR d4, [x6], 8
+ DUP d4, v4.d[1]
+ STR d5, [x8], 8
+ DUP d5, v5.d[1]
+ STR d6, [x9], 8
+ DUP d6, v6.d[1]
+ STR d7, [x7], 8
+ DUP d7, v7.d[1]
+6:
+ TBZ x1, 2, 7f
+ STR s4, [x6], 4
+ DUP s4, v4.s[1]
+ STR s5, [x8], 4
+ DUP s5, v5.s[1]
+ STR s6, [x9], 4
+ DUP s6, v6.s[1]
+ STR s7, [x7], 4
+ DUP s7, v7.s[1]
+7:
+ TBZ x1, 1, 8f
+ ST1 {v4.h}[0], [x6], 2
+ DUP h4, v4.h[1]
+ ST1 {v5.h}[0], [x8], 2
+ DUP h5, v5.h[1]
+ ST1 {v6.h}[0], [x9], 2
+ DUP h6, v6.h[1]
+ ST1 {v7.h}[0], [x7], 2
+ DUP h7, v7.h[1]
+8:
+ TBZ x1, 0, 9f
+ ST1 {v4.b}[0], [x6]
+ ST1 {v5.b}[0], [x8]
+ ST1 {v6.b}[0], [x9]
+ ST1 {v7.b}[0], [x7]
+9:
+ RET
+
+END_FUNCTION xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+
+#ifdef __ELF__
+.section ".note.GNU-stack","",%progbits
+#endif
diff --git a/src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S b/src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
new file mode 100644
index 0000000..69406d4
--- /dev/null
+++ b/src/qs8-gemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
@@ -0,0 +1,794 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-gemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+#include <xnnpack/assembly.h>
+
+# void xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53(
+# size_t mr, x0
+# size_t nc, x1
+# size_t kc, x2 / x0
+# const int8_t* restrict a, x3
+# size_t a_stride, x4
+# const void* restrict w, x5
+# int8_t* restrict c, x6
+# size_t cm_stride, x7
+# size_t cn_stride, [sp] -> x12
+# const union xnn_qs8_conv_minmax_params params) [sp + 8] -> x11
+
+# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
+
+# Register usage
+# A0 x3 v0
+# A1 x15 v1
+# A2 x13 v2
+# A3 x4 v3
+# B x5 v4 v5 v6
+# C0 x6 v16 v20 v24 v28
+# C1 x8 v17 v21 v25 v29
+# C2 x9 v18 v22 v26 v30
+# C3 x7 v19 v23 v27 v31
+# temp v7
+# unused v8 v9 v10 v11 v12 v13 v14 v15
+
+# x10 x17 a53 temp registers
+
+BEGIN_FUNCTION xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+
+ # Clamp A and C pointers
+ CMP x0, 2 // if mr < 2
+ ADD x15, x3, x4 // a1 = a0 + a_stride
+ ADD x8, x6, x7 // c1 = c0 + cm_stride
+ CSEL x15, x3, x15, LO // a1 = a0
+ CSEL x8, x6, x8, LO // c1 = c0
+
+ ADD x13, x15, x4 // a2 = a1 + a_stride
+ ADD x9, x8, x7 // c2 = c1 + cm_stride
+ // if mr <= 2
+ CSEL x13, x15, x13, LS // a2 = a1
+ CSEL x9, x8, x9, LS // c2 = c1
+
+ CMP x0, 4 // if mr < 4
+ ADD x4, x13, x4 // a3 = a2 + a_stride
+ ADD x7, x9, x7 // c3 = c2 + cm_stride
+ CSEL x4, x13, x4, LO // a3 = a2
+ CSEL x7, x9, x7, LO // c3 = c2
+
+ .p2align 3
+0:
+ # Load initial bias from w into accumulators
+ LDP q16, q20, [x5], 32
+ MOV v17.16b, v16.16b
+ MOV v18.16b, v16.16b
+ LDP q24, q28, [x5], 32
+ MOV v19.16b, v16.16b
+ MOV v21.16b, v20.16b
+ LDR x11, [sp, 8] // reload params
+ MOV v22.16b, v20.16b
+ MOV v23.16b, v20.16b
+ SUBS x0, x2, 8 // k = kc - 8
+ MOV v25.16b, v24.16b
+ MOV v26.16b, v24.16b
+ MOV v27.16b, v24.16b
+ MOV v29.16b, v28.16b
+ MOV v30.16b, v28.16b
+ MOV v31.16b, v28.16b
+ # Is there at least 8 bytes for epilogue?
+ B.LO 4f
+
+ # Prologue
+ LDR d0, [x3], 8
+ LDP d4, d6, [x5]
+ LDR d1, [x15], 8
+ LDR d2, [x13], 8
+ LDR d3, [x4], 8
+ SXTL v0.8h, v0.8b
+ LDR x17, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+
+ SUBS x0, x0, 8 // k = k - 8
+ # Is there at least 8 bytes for main loop?
+ B.LO 2f
+
+ # Main loop - 8 bytes of A
+ .p2align 3
+1:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ PRFM PLDL1KEEP, [x3, 128]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ PRFM PLDL1KEEP, [x15, 128]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ PRFM PLDL1KEEP, [x13, 128]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ PRFM PLDL1KEEP, [x4, 128]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ PRFM PLDL1KEEP, [x5, 448]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ PRFM PLDL1KEEP, [x5, 512]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x17, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x17, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x17, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x17, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x17, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ LDR x17, [x5, 112]
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ SXTL v4.8h, v4.8b
+ ADD x5, x5, 128
+
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ LDR x17, [x5]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SXTL v5.8h, v5.8b
+ LDR x10, [x3], 8
+
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ LDR d6, [x5, 8]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ LDR x17, [x13], 8
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ LDR d1, [x15], 8
+ INS v0.d[0], x10
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+ LDR d3, [x4], 8
+ INS v2.d[0], x17
+
+ SXTL v0.8h, v0.8b
+ SXTL v1.8h, v1.8b
+ LDR x17, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v2.8h, v2.8b
+ SUBS x0, x0, 8
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+ B.HS 1b
+
+ # Epilogue. Same as main loop but no preloads in final group
+
+ .p2align 3
+2:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x17, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x17, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x17, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x17
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x17, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x17
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x17, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x17
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR x17, [x5, 112]
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x17
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ ADD x5, x5, 128
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ AND x0, x2, 7 // kc remainder 0 to 7
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+
+ # Is there a remainder?- 1 to 7 bytes of A
+ CBNZ x0, 4f
+
+3:
+ # Apply params - scale, bias and clamp
+ LD1R {v0.4s}, [x11], 4
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v0.4s
+ FMUL v21.4s, v21.4s, v0.4s
+ FMUL v22.4s, v22.4s, v0.4s
+ FMUL v23.4s, v23.4s, v0.4s
+ FMUL v24.4s, v24.4s, v0.4s
+ FMUL v25.4s, v25.4s, v0.4s
+ FMUL v26.4s, v26.4s, v0.4s
+ FMUL v27.4s, v27.4s, v0.4s
+ FMUL v28.4s, v28.4s, v0.4s
+ FMUL v29.4s, v29.4s, v0.4s
+ FMUL v30.4s, v30.4s, v0.4s
+ FMUL v31.4s, v31.4s, v0.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
+
+ SQXTN v16.4h, v16.4s
+ SQXTN v17.4h, v17.4s
+ SQXTN v18.4h, v18.4s
+ SQXTN v19.4h, v19.4s
+ SQXTN v24.4h, v24.4s
+ SQXTN v25.4h, v25.4s
+ SQXTN v26.4h, v26.4s
+ SQXTN v27.4h, v27.4s
+ LD1R {v2.8h}, [x11], 2 // add bias
+
+ SQXTN2 v16.8h, v20.4s
+ SQXTN2 v17.8h, v21.4s
+ SQXTN2 v18.8h, v22.4s
+ SQXTN2 v19.8h, v23.4s
+ SQXTN2 v24.8h, v28.4s
+ SQXTN2 v25.8h, v29.4s
+ SQXTN2 v26.8h, v30.4s
+ SQXTN2 v27.8h, v31.4s
+
+ SQADD v16.8h, v16.8h, v2.8h
+ SQADD v17.8h, v17.8h, v2.8h
+ SQADD v18.8h, v18.8h, v2.8h
+ SQADD v19.8h, v19.8h, v2.8h
+ SQADD v24.8h, v24.8h, v2.8h
+ SQADD v25.8h, v25.8h, v2.8h
+ SQADD v26.8h, v26.8h, v2.8h
+ SQADD v27.8h, v27.8h, v2.8h
+ LD1R {v0.16b}, [x11], 1 // clamp min value
+
+ SQXTN v4.8b, v16.8h
+ SQXTN v5.8b, v17.8h
+ SQXTN v6.8b, v18.8h
+ SQXTN v7.8b, v19.8h
+ LD1R {v1.16b}, [x11] // clamp max value
+ SQXTN2 v4.16b, v24.8h
+ SQXTN2 v5.16b, v25.8h
+ SQXTN2 v6.16b, v26.8h
+ SQXTN2 v7.16b, v27.8h
+ LDR x12, [sp, 0] // cn_stride
+
+ SMAX v4.16b, v4.16b, v0.16b
+ SMAX v5.16b, v5.16b, v0.16b
+ SMAX v6.16b, v6.16b, v0.16b
+ SMAX v7.16b, v7.16b, v0.16b
+ SUBS x1, x1, 16
+ SMIN v4.16b, v4.16b, v1.16b
+ SMIN v5.16b, v5.16b, v1.16b
+ SMIN v6.16b, v6.16b, v1.16b
+ SMIN v7.16b, v7.16b, v1.16b
+ B.LO 5f
+
+ # Store full 4 x 16
+ ST1 {v4.16b}, [x6], x12
+ SUB x3, x3, x2 // a0 -= kc
+ ST1 {v5.16b}, [x8], x12
+ SUB x15, x15, x2 // a1 -= kc
+ ST1 {v6.16b}, [x9], x12
+ SUB x13, x13, x2 // a2 -= kc
+ ST1 {v7.16b}, [x7], x12
+ SUB x4, x4, x2 // a3 -= kc
+ B.NE 0b
+ RET
+
+ # Remainder- 1 to 7 bytes of A
+ .p2align 3
+4:
+ AND x0, x2, 7 // kc remainder 1 to 7
+
+ LD1 {v0.8b}, [x3], x0
+ LDP d4, d5, [x5], 16
+ LD1 {v1.8b}, [x15], x0
+ LD1 {v2.8b}, [x13], x0
+ LD1 {v3.8b}, [x4], x0
+ SXTL v0.8h, v0.8b
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v24.4s, v5.4h, v0.h[0]
+ SMLAL2 v28.4s, v5.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v25.4s, v5.4h, v1.h[0]
+ SMLAL2 v29.4s, v5.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v26.4s, v5.4h, v2.h[0]
+ SMLAL2 v30.4s, v5.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ SMLAL v27.4s, v5.4h, v3.h[0]
+ SMLAL2 v31.4s, v5.8h, v3.h[0]
+ CMP x0, 2
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[1]
+ SMLAL2 v20.4s, v4.8h, v0.h[1]
+ SMLAL v24.4s, v5.4h, v0.h[1]
+ SMLAL2 v28.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v4.4h, v1.h[1]
+ SMLAL2 v21.4s, v4.8h, v1.h[1]
+ SMLAL v25.4s, v5.4h, v1.h[1]
+ SMLAL2 v29.4s, v5.8h, v1.h[1]
+ SMLAL v18.4s, v4.4h, v2.h[1]
+ SMLAL2 v22.4s, v4.8h, v2.h[1]
+ SMLAL v26.4s, v5.4h, v2.h[1]
+ SMLAL2 v30.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v4.4h, v3.h[1]
+ SMLAL2 v23.4s, v4.8h, v3.h[1]
+ SMLAL v27.4s, v5.4h, v3.h[1]
+ SMLAL2 v31.4s, v5.8h, v3.h[1]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[2]
+ SMLAL2 v20.4s, v4.8h, v0.h[2]
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v17.4s, v4.4h, v1.h[2]
+ SMLAL2 v21.4s, v4.8h, v1.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SMLAL v18.4s, v4.4h, v2.h[2]
+ SMLAL2 v22.4s, v4.8h, v2.h[2]
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v19.4s, v4.4h, v3.h[2]
+ SMLAL2 v23.4s, v4.8h, v3.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ CMP x0, 4
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v24.4s, v5.4h, v0.h[3]
+ SMLAL2 v28.4s, v5.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SMLAL v25.4s, v5.4h, v1.h[3]
+ SMLAL2 v29.4s, v5.8h, v1.h[3]
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v26.4s, v5.4h, v2.h[3]
+ SMLAL2 v30.4s, v5.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ SMLAL v27.4s, v5.4h, v3.h[3]
+ SMLAL2 v31.4s, v5.8h, v3.h[3]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[4]
+ SMLAL2 v20.4s, v4.8h, v0.h[4]
+ SMLAL v24.4s, v5.4h, v0.h[4]
+ SMLAL2 v28.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v4.4h, v1.h[4]
+ SMLAL2 v21.4s, v4.8h, v1.h[4]
+ SMLAL v25.4s, v5.4h, v1.h[4]
+ SMLAL2 v29.4s, v5.8h, v1.h[4]
+ SMLAL v18.4s, v4.4h, v2.h[4]
+ SMLAL2 v22.4s, v4.8h, v2.h[4]
+ SMLAL v26.4s, v5.4h, v2.h[4]
+ SMLAL2 v30.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v4.4h, v3.h[4]
+ SMLAL2 v23.4s, v4.8h, v3.h[4]
+ SMLAL v27.4s, v5.4h, v3.h[4]
+ SMLAL2 v31.4s, v5.8h, v3.h[4]
+ CMP x0, 6
+ B.LO 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[5]
+ SMLAL2 v20.4s, v4.8h, v0.h[5]
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v17.4s, v4.4h, v1.h[5]
+ SMLAL2 v21.4s, v4.8h, v1.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SMLAL v18.4s, v4.4h, v2.h[5]
+ SMLAL2 v22.4s, v4.8h, v2.h[5]
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v19.4s, v4.4h, v3.h[5]
+ SMLAL2 v23.4s, v4.8h, v3.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ B.EQ 3b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v24.4s, v5.4h, v0.h[6]
+ SMLAL2 v28.4s, v5.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v25.4s, v5.4h, v1.h[6]
+ SMLAL2 v29.4s, v5.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v26.4s, v5.4h, v2.h[6]
+ SMLAL2 v30.4s, v5.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ SMLAL v27.4s, v5.4h, v3.h[6]
+ SMLAL2 v31.4s, v5.8h, v3.h[6]
+ B 3b
+
+ # Store odd width
+ .p2align 3
+5:
+ TBZ x1, 3, 6f
+ STR d4, [x6], 8
+ DUP d4, v4.d[1]
+ STR d5, [x8], 8
+ DUP d5, v5.d[1]
+ STR d6, [x9], 8
+ DUP d6, v6.d[1]
+ STR d7, [x7], 8
+ DUP d7, v7.d[1]
+6:
+ TBZ x1, 2, 7f
+ STR s4, [x6], 4
+ DUP s4, v4.s[1]
+ STR s5, [x8], 4
+ DUP s5, v5.s[1]
+ STR s6, [x9], 4
+ DUP s6, v6.s[1]
+ STR s7, [x7], 4
+ DUP s7, v7.s[1]
+7:
+ TBZ x1, 1, 8f
+ ST1 {v4.h}[0], [x6], 2
+ DUP h4, v4.h[1]
+ ST1 {v5.h}[0], [x8], 2
+ DUP h5, v5.h[1]
+ ST1 {v6.h}[0], [x9], 2
+ DUP h6, v6.h[1]
+ ST1 {v7.h}[0], [x7], 2
+ DUP h7, v7.h[1]
+8:
+ TBZ x1, 0, 9f
+ ST1 {v4.b}[0], [x6]
+ ST1 {v5.b}[0], [x8]
+ ST1 {v6.b}[0], [x9]
+ ST1 {v7.b}[0], [x7]
+9:
+ RET
+
+END_FUNCTION xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+
+#ifdef __ELF__
+.section ".note.GNU-stack","",%progbits
+#endif
diff --git a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
index b1a3ccd..e12be82 100644
--- a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
+++ b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
@@ -7,6 +7,7 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+
#include <xnnpack/assembly.h>
# void xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53(
@@ -68,7 +69,7 @@
LDP q24, q28, [x5], 32
MOV v19.16b, v16.16b
MOV v21.16b, v20.16b
- LDR x11, [sp, 8] // params
+ LDR x11, [sp, 8] // reload params
MOV v22.16b, v20.16b
MOV v23.16b, v20.16b
SUBS x0, x2, 8 // k = kc - 8
diff --git a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
index a41e00e..e62bb1d 100644
--- a/src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+++ b/src/qs8-gemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
@@ -7,6 +7,7 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+
#include <xnnpack/assembly.h>
# void xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53(
@@ -68,7 +69,7 @@
LDP q24, q28, [x5], 32
MOV v19.16b, v16.16b
MOV v21.16b, v20.16b
- LDR x11, [sp, 8] // params
+ LDR x11, [sp, 8] // reload params
MOV v22.16b, v20.16b
MOV v23.16b, v20.16b
SUBS x0, x2, 8 // k = kc - 8
diff --git a/src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in b/src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
index 6f3b8e0..31de16a 100644
--- a/src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+++ b/src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
@@ -3,9 +3,14 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+$assert REQUANTIZATION in ["GEMMLOWP", "FP32"]
+$assert not CHANNELWISE or REQUANTIZATION == "FP32"
+
#include <xnnpack/assembly.h>
-# void xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53(
+$DATATYPE = "qc8" if CHANNELWISE else "qs8"
+$CONV_PARAMS = "xnn_qs8_minmax_params" if CHANNELWISE else "xnn_qs8_conv_minmax_params"
+# void xnn_${DATATYPE}_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53(
# size_t mr, x0
# size_t nc, x1
# size_t kc, x2 / x0
@@ -35,7 +40,7 @@
# unused v8 v9 v10 v11 v12 v13 v14 v15
# x8, x21 temp for Cortex-A53 loads
-BEGIN_FUNCTION xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53
+BEGIN_FUNCTION xnn_${DATATYPE}_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53
# Clamp C pointers
CMP x0, 2 // if mr < 2
@@ -491,86 +496,163 @@
SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
B.HI 1b
- # Apply params - scale, shift, bias and clamp
- LD2R {v0.4s, v1.4s}, [x8], 8
- CMEQ v2.4s, v1.4s, 0
+ $if REQUANTIZATION == "GEMMLOWP":
+ # Apply params - scale, shift, bias and clamp
+ LD2R {v0.4s, v1.4s}, [x8], 8
+ CMEQ v2.4s, v1.4s, 0
- BIC v4.16b, v16.16b, v2.16b
- BIC v5.16b, v17.16b, v2.16b
- BIC v6.16b, v18.16b, v2.16b
- BIC v7.16b, v19.16b, v2.16b
+ BIC v4.16b, v16.16b, v2.16b
+ BIC v5.16b, v17.16b, v2.16b
+ BIC v6.16b, v18.16b, v2.16b
+ BIC v7.16b, v19.16b, v2.16b
- SQRDMULH v16.4s, v16.4s, v0.4s
- SQRDMULH v17.4s, v17.4s, v0.4s
- SQRDMULH v18.4s, v18.4s, v0.4s
- SQRDMULH v19.4s, v19.4s, v0.4s
+ SQRDMULH v16.4s, v16.4s, v0.4s
+ SQRDMULH v17.4s, v17.4s, v0.4s
+ SQRDMULH v18.4s, v18.4s, v0.4s
+ SQRDMULH v19.4s, v19.4s, v0.4s
- SSRA v16.4s, v4.4s, 31 // signed shift right accumulate
- SSRA v17.4s, v5.4s, 31
- SSRA v18.4s, v6.4s, 31
- SSRA v19.4s, v7.4s, 31
+ SSRA v16.4s, v4.4s, 31 // signed shift right accumulate
+ SSRA v17.4s, v5.4s, 31
+ SSRA v18.4s, v6.4s, 31
+ SSRA v19.4s, v7.4s, 31
- BIC v4.16b, v20.16b, v2.16b
- BIC v5.16b, v21.16b, v2.16b
- BIC v6.16b, v22.16b, v2.16b
- BIC v7.16b, v23.16b, v2.16b
+ BIC v4.16b, v20.16b, v2.16b
+ BIC v5.16b, v21.16b, v2.16b
+ BIC v6.16b, v22.16b, v2.16b
+ BIC v7.16b, v23.16b, v2.16b
- SQRDMULH v20.4s, v20.4s, v0.4s
- SQRDMULH v21.4s, v21.4s, v0.4s
- SQRDMULH v22.4s, v22.4s, v0.4s
- SQRDMULH v23.4s, v23.4s, v0.4s
+ SQRDMULH v20.4s, v20.4s, v0.4s
+ SQRDMULH v21.4s, v21.4s, v0.4s
+ SQRDMULH v22.4s, v22.4s, v0.4s
+ SQRDMULH v23.4s, v23.4s, v0.4s
- SSRA v20.4s, v4.4s, 31
- SSRA v21.4s, v5.4s, 31
- SSRA v22.4s, v6.4s, 31
- SSRA v23.4s, v7.4s, 31
+ SSRA v20.4s, v4.4s, 31
+ SSRA v21.4s, v5.4s, 31
+ SSRA v22.4s, v6.4s, 31
+ SSRA v23.4s, v7.4s, 31
- BIC v4.16b, v24.16b, v2.16b
- BIC v5.16b, v25.16b, v2.16b
- BIC v6.16b, v26.16b, v2.16b
- BIC v7.16b, v27.16b, v2.16b
+ BIC v4.16b, v24.16b, v2.16b
+ BIC v5.16b, v25.16b, v2.16b
+ BIC v6.16b, v26.16b, v2.16b
+ BIC v7.16b, v27.16b, v2.16b
- SQRDMULH v24.4s, v24.4s, v0.4s
- SQRDMULH v25.4s, v25.4s, v0.4s
- SQRDMULH v26.4s, v26.4s, v0.4s
- SQRDMULH v27.4s, v27.4s, v0.4s
+ SQRDMULH v24.4s, v24.4s, v0.4s
+ SQRDMULH v25.4s, v25.4s, v0.4s
+ SQRDMULH v26.4s, v26.4s, v0.4s
+ SQRDMULH v27.4s, v27.4s, v0.4s
- SSRA v24.4s, v4.4s, 31
- SSRA v25.4s, v5.4s, 31
- SSRA v26.4s, v6.4s, 31
- SSRA v27.4s, v7.4s, 31
+ SSRA v24.4s, v4.4s, 31
+ SSRA v25.4s, v5.4s, 31
+ SSRA v26.4s, v6.4s, 31
+ SSRA v27.4s, v7.4s, 31
- BIC v4.16b, v28.16b, v2.16b
- BIC v5.16b, v29.16b, v2.16b
- BIC v6.16b, v30.16b, v2.16b
- BIC v7.16b, v31.16b, v2.16b
+ BIC v4.16b, v28.16b, v2.16b
+ BIC v5.16b, v29.16b, v2.16b
+ BIC v6.16b, v30.16b, v2.16b
+ BIC v7.16b, v31.16b, v2.16b
- SQRDMULH v28.4s, v28.4s, v0.4s
- SQRDMULH v29.4s, v29.4s, v0.4s
- SQRDMULH v30.4s, v30.4s, v0.4s
- SQRDMULH v31.4s, v31.4s, v0.4s
+ SQRDMULH v28.4s, v28.4s, v0.4s
+ SQRDMULH v29.4s, v29.4s, v0.4s
+ SQRDMULH v30.4s, v30.4s, v0.4s
+ SQRDMULH v31.4s, v31.4s, v0.4s
- SSRA v28.4s, v4.4s, 31
- SSRA v29.4s, v5.4s, 31
- SSRA v30.4s, v6.4s, 31
- SSRA v31.4s, v7.4s, 31
+ SSRA v28.4s, v4.4s, 31
+ SSRA v29.4s, v5.4s, 31
+ SSRA v30.4s, v6.4s, 31
+ SSRA v31.4s, v7.4s, 31
- SRSHL v16.4s, v16.4s, v1.4s // signed rounding shift left
- SRSHL v17.4s, v17.4s, v1.4s
- SRSHL v18.4s, v18.4s, v1.4s
- SRSHL v19.4s, v19.4s, v1.4s
- SRSHL v20.4s, v20.4s, v1.4s
- SRSHL v21.4s, v21.4s, v1.4s
- SRSHL v22.4s, v22.4s, v1.4s
- SRSHL v23.4s, v23.4s, v1.4s
- SRSHL v24.4s, v24.4s, v1.4s
- SRSHL v25.4s, v25.4s, v1.4s
- SRSHL v26.4s, v26.4s, v1.4s
- SRSHL v27.4s, v27.4s, v1.4s
- SRSHL v28.4s, v28.4s, v1.4s
- SRSHL v29.4s, v29.4s, v1.4s
- SRSHL v30.4s, v30.4s, v1.4s
- SRSHL v31.4s, v31.4s, v1.4s
+ SRSHL v16.4s, v16.4s, v1.4s // signed rounding shift left
+ SRSHL v17.4s, v17.4s, v1.4s
+ SRSHL v18.4s, v18.4s, v1.4s
+ SRSHL v19.4s, v19.4s, v1.4s
+ SRSHL v20.4s, v20.4s, v1.4s
+ SRSHL v21.4s, v21.4s, v1.4s
+ SRSHL v22.4s, v22.4s, v1.4s
+ SRSHL v23.4s, v23.4s, v1.4s
+ SRSHL v24.4s, v24.4s, v1.4s
+ SRSHL v25.4s, v25.4s, v1.4s
+ SRSHL v26.4s, v26.4s, v1.4s
+ SRSHL v27.4s, v27.4s, v1.4s
+ SRSHL v28.4s, v28.4s, v1.4s
+ SRSHL v29.4s, v29.4s, v1.4s
+ SRSHL v30.4s, v30.4s, v1.4s
+ SRSHL v31.4s, v31.4s, v1.4s
+ $elif REQUANTIZATION == "FP32":
+ $if CHANNELWISE:
+ # Load per channel scale values from weights
+ LDP q0, q1, [x5], 32
+ $else:
+ # Apply params - scale, bias and clamp
+ LD1R {v0.4s}, [x8], 4
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ $if CHANNELWISE:
+ LDP q2, q3, [x5], 32
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v1.4s
+ FMUL v21.4s, v21.4s, v1.4s
+ FMUL v22.4s, v22.4s, v1.4s
+ FMUL v23.4s, v23.4s, v1.4s
+ FMUL v24.4s, v24.4s, v2.4s
+ FMUL v25.4s, v25.4s, v2.4s
+ FMUL v26.4s, v26.4s, v2.4s
+ FMUL v27.4s, v27.4s, v2.4s
+ FMUL v28.4s, v28.4s, v3.4s
+ FMUL v29.4s, v29.4s, v3.4s
+ FMUL v30.4s, v30.4s, v3.4s
+ FMUL v31.4s, v31.4s, v3.4s
+ $else:
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v0.4s
+ FMUL v21.4s, v21.4s, v0.4s
+ FMUL v22.4s, v22.4s, v0.4s
+ FMUL v23.4s, v23.4s, v0.4s
+ FMUL v24.4s, v24.4s, v0.4s
+ FMUL v25.4s, v25.4s, v0.4s
+ FMUL v26.4s, v26.4s, v0.4s
+ FMUL v27.4s, v27.4s, v0.4s
+ FMUL v28.4s, v28.4s, v0.4s
+ FMUL v29.4s, v29.4s, v0.4s
+ FMUL v30.4s, v30.4s, v0.4s
+ FMUL v31.4s, v31.4s, v0.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
SQXTN v16.4h, v16.4s
SQXTN v17.4h, v17.4s
@@ -610,7 +692,13 @@
SQXTN2 v5.16b, v25.8h
SQXTN2 v6.16b, v26.8h
SQXTN2 v7.16b, v27.8h
- SUB x8, x8, 11 // rewind params pointer
+ $if REQUANTIZATION == "GEMMLOWP":
+ SUB x8, x8, 11 // rewind params pointer
+ $elif REQUANTIZATION == "FP32":
+ $if CHANNELWISE:
+ SUB x8, x8, 3 // rewind params pointer
+ $else:
+ SUB x8, x8, 7 // rewind params pointer
SMAX v4.16b, v4.16b, v0.16b
SMAX v5.16b, v5.16b, v0.16b
@@ -844,7 +932,7 @@
LDP x20, x21, [sp], 16
RET
-END_FUNCTION xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53
+END_FUNCTION xnn_${DATATYPE}_igemm_minmax_${REQUANTIZATION.lower()}_ukernel_4x16__aarch64_neon_mlal_lane${"_prfm" if PREFETCH else ""}_cortex_a53
#ifdef __ELF__
.section ".note.GNU-stack","",%progbits
diff --git a/src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S b/src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
new file mode 100644
index 0000000..b41e45b
--- /dev/null
+++ b/src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-cortex-a53.S
@@ -0,0 +1,816 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+// Generator: tools/xngen
+//
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+#include <xnnpack/assembly.h>
+
+# void xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53(
+# size_t mr, x0
+# size_t nc, x1
+# size_t kc, x2 / x0
+# size_t ks, x3 / x9
+# const int8_t**restrict a, x4
+# const int8_t* restrict w, x5
+# int8_t* restrict c, x6
+# size_t cm_stride, x7
+# size_t cn_stride, [sp] -> x10
+# size_t a_offset, [sp + 8] -> x11
+# const float* zero, [sp + 16] -> x12
+# const xnn_qs8_conv_minmax_params params [sp + 24] -> (x8)
+
+# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
+
+# Register usage
+# A0 x13 v0
+# A1 x14 v1
+# A2 x15 v2
+# A3 x20 v3
+# B x5 v4 v5 v6
+# C0 x6 v16 v20 v24 v28
+# C1 x16 v17 v21 v25 v29
+# C2 x17 v18 v22 v26 v30
+# C3 x7 v19 v23 v27 v31
+# temp v7
+# unused v8 v9 v10 v11 v12 v13 v14 v15
+# x8, x21 temp for Cortex-A53 loads
+
+BEGIN_FUNCTION xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+
+ # Clamp C pointers
+ CMP x0, 2 // if mr < 2
+ LDP x10, x11, [sp] // Load cn_stride, a_offset
+ ADD x16, x6, x7 // c1 = c0 + cm_stride
+ CSEL x16, x6, x16, LO // c1 = c0
+
+ ADD x17, x16, x7 // c2 = c1 + cm_stride
+ LDP x12, x8, [sp, 16] // Load zero, params pointer
+ // if mr <= 2
+ CSEL x17, x16, x17, LS // c2 = c1
+
+ CMP x0, 4 // if mr < 4
+ STP x20, x21, [sp, -16]! // Save x20-x21 on stack
+ ADD x7, x17, x7 // c3 = c2 + cm_stride
+ CSEL x7, x17, x7, LO // c3 = c2
+
+ .p2align 3
+0:
+ # Load initial bias from w into accumulators
+ LDP q16, q20, [x5], 32
+ MOV v17.16b, v16.16b
+ MOV v18.16b, v16.16b
+ LDP q24, q28, [x5], 32
+ MOV v19.16b, v16.16b
+ MOV v21.16b, v20.16b
+ MOV v22.16b, v20.16b
+ MOV v23.16b, v20.16b
+ MOV v25.16b, v24.16b
+ MOV v26.16b, v24.16b
+ MOV v27.16b, v24.16b
+ MOV v29.16b, v28.16b
+ MOV v30.16b, v28.16b
+ MOV v31.16b, v28.16b
+ MOV x9, x3 // p = ks
+
+ .p2align 3
+1:
+ # Load next 4 A pointers
+ LDP x13, x14, [x4], 16
+ LDP x15, x20, [x4], 16
+
+ CMP x13, x12 // if a0 == zero
+ ADD x13, x13, x11 // a0 += a_offset
+ CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
+ CMP x14, x12 // if a1 == zero
+ ADD x14, x14, x11 // a1 += a_offset
+ CSEL x14, x12, x14, EQ // a1 = zero, else += a1 + a_offset
+ CMP x15, x12 // if a2 == zero
+ ADD x15, x15, x11 // a2 += a_offset
+ CSEL x15, x12, x15, EQ // a2 = zero, else += a2 + a_offset
+ CMP x20, x12 // if a3 == zero
+ ADD x20, x20, x11 // a3 += a_offset
+ CSEL x20, x12, x20, EQ // a3 = zero, else += a3 + a_offset
+
+ # Is there at least 8 bytes for epilogue?
+ SUBS x0, x2, 8 // k = kc - 8
+ B.LO 5f
+
+ # Prologue
+ LDR d0, [x13], 8
+ LDP d4, d6, [x5]
+ LDR d1, [x14], 8
+ LDR d2, [x15], 8
+ LDR d3, [x20], 8
+ SXTL v0.8h, v0.8b
+ LDR x8, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+
+ SUBS x0, x0, 8 // k = k - 8
+ # Is there at least 8 bytes for main loop?
+ B.LO 3f
+
+ # Main loop - 8 bytes of A
+ .p2align 3
+2:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x8, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x8, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x8, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x8, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x8, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ LDR x8, [x5, 112]
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ SXTL v4.8h, v4.8b
+ ADD x5, x5, 128
+
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ LDR x8, [x5]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SXTL v5.8h, v5.8b
+ LDR x21, [x13], 8
+
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ LDR d6, [x5, 8]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ LDR x8, [x15], 8
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ LDR d1, [x14], 8
+ INS v0.d[0], x21
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+ LDR d3, [x20], 8
+ INS v2.d[0], x8
+
+ SXTL v0.8h, v0.8b
+ SXTL v1.8h, v1.8b
+ LDR x8, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v2.8h, v2.8b
+ SUBS x0, x0, 8
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+ B.HS 2b
+
+ # Epilogue. Same as main loop but no preloads in final group
+
+ .p2align 3
+3:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x8, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x8, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x8, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x8, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x8, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR x8, [x5, 112]
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x8
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ ADD x5, x5, 128
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ AND x0, x2, 7 // kc remainder 0 to 7
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ LDR x8, [sp, 40] // reload params pointer
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+
+ # Is there a remainder?- 1 to 7 bytes of A
+ CBNZ x0, 5f
+
+4:
+ # ks loop
+ SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
+ B.HI 1b
+
+ # Apply params - scale, bias and clamp
+ LD1R {v0.4s}, [x8], 4
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v0.4s
+ FMUL v21.4s, v21.4s, v0.4s
+ FMUL v22.4s, v22.4s, v0.4s
+ FMUL v23.4s, v23.4s, v0.4s
+ FMUL v24.4s, v24.4s, v0.4s
+ FMUL v25.4s, v25.4s, v0.4s
+ FMUL v26.4s, v26.4s, v0.4s
+ FMUL v27.4s, v27.4s, v0.4s
+ FMUL v28.4s, v28.4s, v0.4s
+ FMUL v29.4s, v29.4s, v0.4s
+ FMUL v30.4s, v30.4s, v0.4s
+ FMUL v31.4s, v31.4s, v0.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
+
+ SQXTN v16.4h, v16.4s
+ SQXTN v17.4h, v17.4s
+ SQXTN v18.4h, v18.4s
+ SQXTN v19.4h, v19.4s
+ SQXTN v24.4h, v24.4s
+ SQXTN v25.4h, v25.4s
+ SQXTN v26.4h, v26.4s
+ SQXTN v27.4h, v27.4s
+ LD1R {v2.8h}, [x8], 2 // add bias
+
+ SQXTN2 v16.8h, v20.4s
+ SQXTN2 v17.8h, v21.4s
+ SQXTN2 v18.8h, v22.4s
+ SQXTN2 v19.8h, v23.4s
+ SQXTN2 v24.8h, v28.4s
+ SQXTN2 v25.8h, v29.4s
+ SQXTN2 v26.8h, v30.4s
+ SQXTN2 v27.8h, v31.4s
+
+ SQADD v16.8h, v16.8h, v2.8h
+ SQADD v17.8h, v17.8h, v2.8h
+ SQADD v18.8h, v18.8h, v2.8h
+ SQADD v19.8h, v19.8h, v2.8h
+ SQADD v24.8h, v24.8h, v2.8h
+ SQADD v25.8h, v25.8h, v2.8h
+ SQADD v26.8h, v26.8h, v2.8h
+ SQADD v27.8h, v27.8h, v2.8h
+ LD1R {v0.16b}, [x8], 1 // clamp min value
+
+ SQXTN v4.8b, v16.8h
+ SQXTN v5.8b, v17.8h
+ SQXTN v6.8b, v18.8h
+ SQXTN v7.8b, v19.8h
+ LD1R {v1.16b}, [x8] // clamp max value
+ SQXTN2 v4.16b, v24.8h
+ SQXTN2 v5.16b, v25.8h
+ SQXTN2 v6.16b, v26.8h
+ SQXTN2 v7.16b, v27.8h
+ SUB x8, x8, 7 // rewind params pointer
+
+ SMAX v4.16b, v4.16b, v0.16b
+ SMAX v5.16b, v5.16b, v0.16b
+ SMAX v6.16b, v6.16b, v0.16b
+ SMAX v7.16b, v7.16b, v0.16b
+ SUBS x1, x1, 16
+ SMIN v4.16b, v4.16b, v1.16b
+ SMIN v5.16b, v5.16b, v1.16b
+ SMIN v6.16b, v6.16b, v1.16b
+ SMIN v7.16b, v7.16b, v1.16b
+ B.LO 6f
+
+ # Store full 4 x 16
+ ST1 {v7.16b}, [x7], x10
+ ST1 {v6.16b}, [x17], x10
+ ST1 {v5.16b}, [x16], x10
+ ST1 {v4.16b}, [x6], x10
+
+ SUB x4, x4, x3 // a -= ks
+
+ # nc loop
+ B.HI 0b
+
+ # Restore x20-x21 from stack
+ LDP x20, x21, [sp], 16
+ RET
+
+ # Remainder- 1 to 7 bytes of A
+ .p2align 3
+5:
+ AND x0, x2, 7 // kc remainder 1 to 7
+
+ LD1 {v0.8b}, [x13], x0
+ LDP d4, d5, [x5], 16
+ LD1 {v1.8b}, [x14], x0
+ LD1 {v2.8b}, [x15], x0
+ LD1 {v3.8b}, [x20], x0
+ SXTL v0.8h, v0.8b
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v24.4s, v5.4h, v0.h[0]
+ SMLAL2 v28.4s, v5.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v25.4s, v5.4h, v1.h[0]
+ SMLAL2 v29.4s, v5.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v26.4s, v5.4h, v2.h[0]
+ SMLAL2 v30.4s, v5.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ SMLAL v27.4s, v5.4h, v3.h[0]
+ SMLAL2 v31.4s, v5.8h, v3.h[0]
+ CMP x0, 2
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[1]
+ SMLAL2 v20.4s, v4.8h, v0.h[1]
+ SMLAL v24.4s, v5.4h, v0.h[1]
+ SMLAL2 v28.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v4.4h, v1.h[1]
+ SMLAL2 v21.4s, v4.8h, v1.h[1]
+ SMLAL v25.4s, v5.4h, v1.h[1]
+ SMLAL2 v29.4s, v5.8h, v1.h[1]
+ SMLAL v18.4s, v4.4h, v2.h[1]
+ SMLAL2 v22.4s, v4.8h, v2.h[1]
+ SMLAL v26.4s, v5.4h, v2.h[1]
+ SMLAL2 v30.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v4.4h, v3.h[1]
+ SMLAL2 v23.4s, v4.8h, v3.h[1]
+ SMLAL v27.4s, v5.4h, v3.h[1]
+ SMLAL2 v31.4s, v5.8h, v3.h[1]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[2]
+ SMLAL2 v20.4s, v4.8h, v0.h[2]
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v17.4s, v4.4h, v1.h[2]
+ SMLAL2 v21.4s, v4.8h, v1.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SMLAL v18.4s, v4.4h, v2.h[2]
+ SMLAL2 v22.4s, v4.8h, v2.h[2]
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v19.4s, v4.4h, v3.h[2]
+ SMLAL2 v23.4s, v4.8h, v3.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ CMP x0, 4
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v24.4s, v5.4h, v0.h[3]
+ SMLAL2 v28.4s, v5.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SMLAL v25.4s, v5.4h, v1.h[3]
+ SMLAL2 v29.4s, v5.8h, v1.h[3]
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v26.4s, v5.4h, v2.h[3]
+ SMLAL2 v30.4s, v5.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ SMLAL v27.4s, v5.4h, v3.h[3]
+ SMLAL2 v31.4s, v5.8h, v3.h[3]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[4]
+ SMLAL2 v20.4s, v4.8h, v0.h[4]
+ SMLAL v24.4s, v5.4h, v0.h[4]
+ SMLAL2 v28.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v4.4h, v1.h[4]
+ SMLAL2 v21.4s, v4.8h, v1.h[4]
+ SMLAL v25.4s, v5.4h, v1.h[4]
+ SMLAL2 v29.4s, v5.8h, v1.h[4]
+ SMLAL v18.4s, v4.4h, v2.h[4]
+ SMLAL2 v22.4s, v4.8h, v2.h[4]
+ SMLAL v26.4s, v5.4h, v2.h[4]
+ SMLAL2 v30.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v4.4h, v3.h[4]
+ SMLAL2 v23.4s, v4.8h, v3.h[4]
+ SMLAL v27.4s, v5.4h, v3.h[4]
+ SMLAL2 v31.4s, v5.8h, v3.h[4]
+ CMP x0, 6
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[5]
+ SMLAL2 v20.4s, v4.8h, v0.h[5]
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v17.4s, v4.4h, v1.h[5]
+ SMLAL2 v21.4s, v4.8h, v1.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SMLAL v18.4s, v4.4h, v2.h[5]
+ SMLAL2 v22.4s, v4.8h, v2.h[5]
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v19.4s, v4.4h, v3.h[5]
+ SMLAL2 v23.4s, v4.8h, v3.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v24.4s, v5.4h, v0.h[6]
+ SMLAL2 v28.4s, v5.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v25.4s, v5.4h, v1.h[6]
+ SMLAL2 v29.4s, v5.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v26.4s, v5.4h, v2.h[6]
+ SMLAL2 v30.4s, v5.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ SMLAL v27.4s, v5.4h, v3.h[6]
+ SMLAL2 v31.4s, v5.8h, v3.h[6]
+ B 4b
+
+ # Store odd width
+ .p2align 3
+6:
+ TBZ x1, 3, 7f
+ STR d7, [x7], 8
+ DUP d7, v7.d[1]
+ STR d6, [x17], 8
+ DUP d6, v6.d[1]
+ STR d5, [x16], 8
+ DUP d5, v5.d[1]
+ STR d4, [x6], 8
+ DUP d4, v4.d[1]
+7:
+ TBZ x1, 2, 8f
+ STR s7, [x7], 4
+ DUP s7, v7.s[1]
+ STR s6, [x17], 4
+ DUP s6, v6.s[1]
+ STR s5, [x16], 4
+ DUP s5, v5.s[1]
+ STR s4, [x6], 4
+ DUP s4, v4.s[1]
+8:
+ TBZ x1, 1, 9f
+ ST1 {v7.h}[0], [x7], 2
+ DUP h7, v7.h[1]
+ ST1 {v6.h}[0], [x17], 2
+ DUP h6, v6.h[1]
+ ST1 {v5.h}[0], [x16], 2
+ DUP h5, v5.h[1]
+ ST1 {v4.h}[0], [x6], 2
+ DUP h4, v4.h[1]
+9:
+ TBZ x1, 0, 10f
+ ST1 {v7.b}[0], [x7]
+ ST1 {v6.b}[0], [x17]
+ ST1 {v5.b}[0], [x16]
+ ST1 {v4.b}[0], [x6]
+10:
+ # Restore x20-x21 from stack
+ LDP x20, x21, [sp], 16
+ RET
+
+END_FUNCTION xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+
+#ifdef __ELF__
+.section ".note.GNU-stack","",%progbits
+#endif
diff --git a/src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S b/src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
new file mode 100644
index 0000000..0d3d6ec
--- /dev/null
+++ b/src/qs8-igemm/gen/4x16-minmax-fp32-aarch64-neon-mlal-lane-prfm-cortex-a53.S
@@ -0,0 +1,822 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-igemm/4x16-aarch64-neon-mlal-lane-cortex-a53.S.in
+// Generator: tools/xngen
+//
+// Copyright 2021 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+
+#include <xnnpack/assembly.h>
+
+# void xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53(
+# size_t mr, x0
+# size_t nc, x1
+# size_t kc, x2 / x0
+# size_t ks, x3 / x9
+# const int8_t**restrict a, x4
+# const int8_t* restrict w, x5
+# int8_t* restrict c, x6
+# size_t cm_stride, x7
+# size_t cn_stride, [sp] -> x10
+# size_t a_offset, [sp + 8] -> x11
+# const float* zero, [sp + 16] -> x12
+# const xnn_qs8_conv_minmax_params params [sp + 24] -> (x8)
+
+# d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
+
+# Register usage
+# A0 x13 v0
+# A1 x14 v1
+# A2 x15 v2
+# A3 x20 v3
+# B x5 v4 v5 v6
+# C0 x6 v16 v20 v24 v28
+# C1 x16 v17 v21 v25 v29
+# C2 x17 v18 v22 v26 v30
+# C3 x7 v19 v23 v27 v31
+# temp v7
+# unused v8 v9 v10 v11 v12 v13 v14 v15
+# x8, x21 temp for Cortex-A53 loads
+
+BEGIN_FUNCTION xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+
+ # Clamp C pointers
+ CMP x0, 2 // if mr < 2
+ LDP x10, x11, [sp] // Load cn_stride, a_offset
+ ADD x16, x6, x7 // c1 = c0 + cm_stride
+ CSEL x16, x6, x16, LO // c1 = c0
+
+ ADD x17, x16, x7 // c2 = c1 + cm_stride
+ LDP x12, x8, [sp, 16] // Load zero, params pointer
+ // if mr <= 2
+ CSEL x17, x16, x17, LS // c2 = c1
+
+ CMP x0, 4 // if mr < 4
+ STP x20, x21, [sp, -16]! // Save x20-x21 on stack
+ ADD x7, x17, x7 // c3 = c2 + cm_stride
+ CSEL x7, x17, x7, LO // c3 = c2
+
+ .p2align 3
+0:
+ # Load initial bias from w into accumulators
+ LDP q16, q20, [x5], 32
+ MOV v17.16b, v16.16b
+ MOV v18.16b, v16.16b
+ LDP q24, q28, [x5], 32
+ MOV v19.16b, v16.16b
+ MOV v21.16b, v20.16b
+ MOV v22.16b, v20.16b
+ MOV v23.16b, v20.16b
+ MOV v25.16b, v24.16b
+ MOV v26.16b, v24.16b
+ MOV v27.16b, v24.16b
+ MOV v29.16b, v28.16b
+ MOV v30.16b, v28.16b
+ MOV v31.16b, v28.16b
+ MOV x9, x3 // p = ks
+
+ .p2align 3
+1:
+ # Load next 4 A pointers
+ LDP x13, x14, [x4], 16
+ LDP x15, x20, [x4], 16
+
+ CMP x13, x12 // if a0 == zero
+ ADD x13, x13, x11 // a0 += a_offset
+ CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
+ CMP x14, x12 // if a1 == zero
+ ADD x14, x14, x11 // a1 += a_offset
+ CSEL x14, x12, x14, EQ // a1 = zero, else += a1 + a_offset
+ CMP x15, x12 // if a2 == zero
+ ADD x15, x15, x11 // a2 += a_offset
+ CSEL x15, x12, x15, EQ // a2 = zero, else += a2 + a_offset
+ CMP x20, x12 // if a3 == zero
+ ADD x20, x20, x11 // a3 += a_offset
+ CSEL x20, x12, x20, EQ // a3 = zero, else += a3 + a_offset
+
+ # Is there at least 8 bytes for epilogue?
+ SUBS x0, x2, 8 // k = kc - 8
+ B.LO 5f
+
+ # Prologue
+ LDR d0, [x13], 8
+ LDP d4, d6, [x5]
+ LDR d1, [x14], 8
+ LDR d2, [x15], 8
+ LDR d3, [x20], 8
+ SXTL v0.8h, v0.8b
+ LDR x8, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+
+ SUBS x0, x0, 8 // k = k - 8
+ # Is there at least 8 bytes for main loop?
+ B.LO 3f
+
+ # Main loop - 8 bytes of A
+ .p2align 3
+2:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ PRFM PLDL1KEEP, [x13, 128]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ PRFM PLDL1KEEP, [x14, 128]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ PRFM PLDL1KEEP, [x15, 128]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ PRFM PLDL1KEEP, [x20, 128]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ PRFM PLDL1KEEP, [x5, 448]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ PRFM PLDL1KEEP, [x5, 512]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x8, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x8, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x8, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x8, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x8, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ LDR x8, [x5, 112]
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ SXTL v4.8h, v4.8b
+ ADD x5, x5, 128
+
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ LDR x8, [x5]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SXTL v5.8h, v5.8b
+ LDR x21, [x13], 8
+
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ LDR d6, [x5, 8]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ LDR x8, [x15], 8
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ LDR d1, [x14], 8
+ INS v0.d[0], x21
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+ LDR d3, [x20], 8
+ INS v2.d[0], x8
+
+ SXTL v0.8h, v0.8b
+ SXTL v1.8h, v1.8b
+ LDR x8, [x5, 16]
+ SXTL v4.8h, v4.8b
+ SXTL v2.8h, v2.8b
+ SUBS x0, x0, 8
+ SXTL v3.8h, v3.8b
+ SXTL v6.8h, v6.8b
+ B.HS 2b
+
+ # Epilogue. Same as main loop but no preloads in final group
+
+ .p2align 3
+3:
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ LDR d4, [x5, 24]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[0]
+ SMLAL2 v28.4s, v6.8h, v0.h[0]
+ SMLAL v25.4s, v6.4h, v1.h[0]
+ SMLAL2 v29.4s, v6.8h, v1.h[0]
+ SXTL v5.8h, v5.8b
+ SMLAL v26.4s, v6.4h, v2.h[0]
+ SMLAL2 v30.4s, v6.8h, v2.h[0]
+ SMLAL v27.4s, v6.4h, v3.h[0]
+ SMLAL2 v31.4s, v6.8h, v3.h[0]
+ LDR x8, [x5, 32]
+ SMLAL v16.4s, v5.4h, v0.h[1]
+ SMLAL2 v20.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v5.4h, v1.h[1]
+ SMLAL2 v21.4s, v5.8h, v1.h[1]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[1]
+ SMLAL2 v22.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v5.4h, v3.h[1]
+ SMLAL2 v23.4s, v5.8h, v3.h[1]
+ LDR d5, [x5, 40]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[1]
+ SMLAL2 v28.4s, v4.8h, v0.h[1]
+ SMLAL v25.4s, v4.4h, v1.h[1]
+ SMLAL2 v29.4s, v4.8h, v1.h[1]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[1]
+ SMLAL2 v30.4s, v4.8h, v2.h[1]
+ SMLAL v27.4s, v4.4h, v3.h[1]
+ SMLAL2 v31.4s, v4.8h, v3.h[1]
+ LDR x8, [x5, 48]
+ SMLAL v16.4s, v6.4h, v0.h[2]
+ SMLAL2 v20.4s, v6.8h, v0.h[2]
+ SMLAL v17.4s, v6.4h, v1.h[2]
+ SXTL v5.8h, v5.8b
+ SMLAL2 v21.4s, v6.8h, v1.h[2]
+ SMLAL v18.4s, v6.4h, v2.h[2]
+ SMLAL2 v22.4s, v6.8h, v2.h[2]
+ SMLAL v19.4s, v6.4h, v3.h[2]
+ SMLAL2 v23.4s, v6.8h, v3.h[2]
+ LDR d6, [x5, 56]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ LDR x8, [x5, 64]
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SXTL v6.8h, v6.8b
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ LDR d4, [x5, 72]
+ INS v5.d[0], x8
+ SMLAL v24.4s, v6.4h, v0.h[3]
+ SMLAL2 v28.4s, v6.8h, v0.h[3]
+ SXTL v5.8h, v5.8b
+ SMLAL v25.4s, v6.4h, v1.h[3]
+ SMLAL2 v29.4s, v6.8h, v1.h[3]
+ SMLAL v26.4s, v6.4h, v2.h[3]
+ SMLAL2 v30.4s, v6.8h, v2.h[3]
+ SMLAL v27.4s, v6.4h, v3.h[3]
+ SMLAL2 v31.4s, v6.8h, v3.h[3]
+ LDR x8, [x5, 80]
+ SMLAL v16.4s, v5.4h, v0.h[4]
+ SMLAL2 v20.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v5.4h, v1.h[4]
+ SMLAL2 v21.4s, v5.8h, v1.h[4]
+ SXTL v4.8h, v4.8b
+ SMLAL v18.4s, v5.4h, v2.h[4]
+ SMLAL2 v22.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v5.4h, v3.h[4]
+ SMLAL2 v23.4s, v5.8h, v3.h[4]
+ LDR d5, [x5, 88]
+ INS v6.d[0], x8
+ SMLAL v24.4s, v4.4h, v0.h[4]
+ SMLAL2 v28.4s, v4.8h, v0.h[4]
+ SMLAL v25.4s, v4.4h, v1.h[4]
+ SMLAL2 v29.4s, v4.8h, v1.h[4]
+ SXTL v6.8h, v6.8b
+ SMLAL v26.4s, v4.4h, v2.h[4]
+ SMLAL2 v30.4s, v4.8h, v2.h[4]
+ SMLAL v27.4s, v4.4h, v3.h[4]
+ SMLAL2 v31.4s, v4.8h, v3.h[4]
+ LDR x8, [x5, 96]
+ SMLAL v16.4s, v6.4h, v0.h[5]
+ SMLAL2 v20.4s, v6.8h, v0.h[5]
+ SMLAL v17.4s, v6.4h, v1.h[5]
+ SMLAL2 v21.4s, v6.8h, v1.h[5]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v6.4h, v2.h[5]
+ SMLAL2 v22.4s, v6.8h, v2.h[5]
+ SMLAL v19.4s, v6.4h, v3.h[5]
+ SMLAL2 v23.4s, v6.8h, v3.h[5]
+ LDR d6, [x5, 104]
+ INS v4.d[0], x8
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ SXTL v6.8h, v6.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ LDR x8, [x5, 112]
+ SMLAL v24.4s, v6.4h, v0.h[6]
+ SMLAL2 v28.4s, v6.8h, v0.h[6]
+ SMLAL v25.4s, v6.4h, v1.h[6]
+ SMLAL2 v29.4s, v6.8h, v1.h[6]
+ LDR d5, [x5, 120]
+ INS v4.d[0], x8
+ SXTL v4.8h, v4.8b
+ SMLAL v26.4s, v6.4h, v2.h[6]
+ SMLAL2 v30.4s, v6.8h, v2.h[6]
+ SMLAL v27.4s, v6.4h, v3.h[6]
+ SMLAL2 v31.4s, v6.8h, v3.h[6]
+ SMLAL v16.4s, v4.4h, v0.h[7]
+ SMLAL2 v20.4s, v4.8h, v0.h[7]
+ SMLAL v17.4s, v4.4h, v1.h[7]
+ SMLAL2 v21.4s, v4.8h, v1.h[7]
+ SXTL v5.8h, v5.8b
+ SMLAL v18.4s, v4.4h, v2.h[7]
+ SMLAL2 v22.4s, v4.8h, v2.h[7]
+ SMLAL v19.4s, v4.4h, v3.h[7]
+ SMLAL2 v23.4s, v4.8h, v3.h[7]
+ ADD x5, x5, 128
+ SMLAL v24.4s, v5.4h, v0.h[7]
+ SMLAL2 v28.4s, v5.8h, v0.h[7]
+ SMLAL v25.4s, v5.4h, v1.h[7]
+ SMLAL2 v29.4s, v5.8h, v1.h[7]
+ AND x0, x2, 7 // kc remainder 0 to 7
+ SMLAL v26.4s, v5.4h, v2.h[7]
+ SMLAL2 v30.4s, v5.8h, v2.h[7]
+ LDR x8, [sp, 40] // reload params pointer
+ SMLAL v27.4s, v5.4h, v3.h[7]
+ SMLAL2 v31.4s, v5.8h, v3.h[7]
+
+ # Is there a remainder?- 1 to 7 bytes of A
+ CBNZ x0, 5f
+
+4:
+ # ks loop
+ SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
+ B.HI 1b
+
+ # Apply params - scale, bias and clamp
+ LD1R {v0.4s}, [x8], 4
+ SCVTF v16.4s, v16.4s
+ SCVTF v17.4s, v17.4s
+ SCVTF v18.4s, v18.4s
+ SCVTF v19.4s, v19.4s
+ SCVTF v20.4s, v20.4s
+ SCVTF v21.4s, v21.4s
+ SCVTF v22.4s, v22.4s
+ SCVTF v23.4s, v23.4s
+ SCVTF v24.4s, v24.4s
+ SCVTF v25.4s, v25.4s
+ SCVTF v26.4s, v26.4s
+ SCVTF v27.4s, v27.4s
+ SCVTF v28.4s, v28.4s
+ SCVTF v29.4s, v29.4s
+ SCVTF v30.4s, v30.4s
+ SCVTF v31.4s, v31.4s
+
+ FMUL v16.4s, v16.4s, v0.4s
+ FMUL v17.4s, v17.4s, v0.4s
+ FMUL v18.4s, v18.4s, v0.4s
+ FMUL v19.4s, v19.4s, v0.4s
+ FMUL v20.4s, v20.4s, v0.4s
+ FMUL v21.4s, v21.4s, v0.4s
+ FMUL v22.4s, v22.4s, v0.4s
+ FMUL v23.4s, v23.4s, v0.4s
+ FMUL v24.4s, v24.4s, v0.4s
+ FMUL v25.4s, v25.4s, v0.4s
+ FMUL v26.4s, v26.4s, v0.4s
+ FMUL v27.4s, v27.4s, v0.4s
+ FMUL v28.4s, v28.4s, v0.4s
+ FMUL v29.4s, v29.4s, v0.4s
+ FMUL v30.4s, v30.4s, v0.4s
+ FMUL v31.4s, v31.4s, v0.4s
+
+ FCVTNS v16.4s, v16.4s
+ FCVTNS v17.4s, v17.4s
+ FCVTNS v18.4s, v18.4s
+ FCVTNS v19.4s, v19.4s
+ FCVTNS v20.4s, v20.4s
+ FCVTNS v21.4s, v21.4s
+ FCVTNS v22.4s, v22.4s
+ FCVTNS v23.4s, v23.4s
+ FCVTNS v24.4s, v24.4s
+ FCVTNS v25.4s, v25.4s
+ FCVTNS v26.4s, v26.4s
+ FCVTNS v27.4s, v27.4s
+ FCVTNS v28.4s, v28.4s
+ FCVTNS v29.4s, v29.4s
+ FCVTNS v30.4s, v30.4s
+ FCVTNS v31.4s, v31.4s
+
+ SQXTN v16.4h, v16.4s
+ SQXTN v17.4h, v17.4s
+ SQXTN v18.4h, v18.4s
+ SQXTN v19.4h, v19.4s
+ SQXTN v24.4h, v24.4s
+ SQXTN v25.4h, v25.4s
+ SQXTN v26.4h, v26.4s
+ SQXTN v27.4h, v27.4s
+ LD1R {v2.8h}, [x8], 2 // add bias
+
+ SQXTN2 v16.8h, v20.4s
+ SQXTN2 v17.8h, v21.4s
+ SQXTN2 v18.8h, v22.4s
+ SQXTN2 v19.8h, v23.4s
+ SQXTN2 v24.8h, v28.4s
+ SQXTN2 v25.8h, v29.4s
+ SQXTN2 v26.8h, v30.4s
+ SQXTN2 v27.8h, v31.4s
+
+ SQADD v16.8h, v16.8h, v2.8h
+ SQADD v17.8h, v17.8h, v2.8h
+ SQADD v18.8h, v18.8h, v2.8h
+ SQADD v19.8h, v19.8h, v2.8h
+ SQADD v24.8h, v24.8h, v2.8h
+ SQADD v25.8h, v25.8h, v2.8h
+ SQADD v26.8h, v26.8h, v2.8h
+ SQADD v27.8h, v27.8h, v2.8h
+ LD1R {v0.16b}, [x8], 1 // clamp min value
+
+ SQXTN v4.8b, v16.8h
+ SQXTN v5.8b, v17.8h
+ SQXTN v6.8b, v18.8h
+ SQXTN v7.8b, v19.8h
+ LD1R {v1.16b}, [x8] // clamp max value
+ SQXTN2 v4.16b, v24.8h
+ SQXTN2 v5.16b, v25.8h
+ SQXTN2 v6.16b, v26.8h
+ SQXTN2 v7.16b, v27.8h
+ SUB x8, x8, 7 // rewind params pointer
+
+ SMAX v4.16b, v4.16b, v0.16b
+ SMAX v5.16b, v5.16b, v0.16b
+ SMAX v6.16b, v6.16b, v0.16b
+ SMAX v7.16b, v7.16b, v0.16b
+ SUBS x1, x1, 16
+ SMIN v4.16b, v4.16b, v1.16b
+ SMIN v5.16b, v5.16b, v1.16b
+ SMIN v6.16b, v6.16b, v1.16b
+ SMIN v7.16b, v7.16b, v1.16b
+ B.LO 6f
+
+ # Store full 4 x 16
+ ST1 {v7.16b}, [x7], x10
+ ST1 {v6.16b}, [x17], x10
+ ST1 {v5.16b}, [x16], x10
+ ST1 {v4.16b}, [x6], x10
+
+ SUB x4, x4, x3 // a -= ks
+
+ # nc loop
+ B.HI 0b
+
+ # Restore x20-x21 from stack
+ LDP x20, x21, [sp], 16
+ RET
+
+ # Remainder- 1 to 7 bytes of A
+ .p2align 3
+5:
+ AND x0, x2, 7 // kc remainder 1 to 7
+
+ LD1 {v0.8b}, [x13], x0
+ LDP d4, d5, [x5], 16
+ LD1 {v1.8b}, [x14], x0
+ LD1 {v2.8b}, [x15], x0
+ LD1 {v3.8b}, [x20], x0
+ SXTL v0.8h, v0.8b
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SXTL v1.8h, v1.8b
+ SXTL v2.8h, v2.8b
+ SXTL v3.8h, v3.8b
+ SMLAL v16.4s, v4.4h, v0.h[0]
+ SMLAL2 v20.4s, v4.8h, v0.h[0]
+ SMLAL v24.4s, v5.4h, v0.h[0]
+ SMLAL2 v28.4s, v5.8h, v0.h[0]
+ SMLAL v17.4s, v4.4h, v1.h[0]
+ SMLAL2 v21.4s, v4.8h, v1.h[0]
+ SMLAL v25.4s, v5.4h, v1.h[0]
+ SMLAL2 v29.4s, v5.8h, v1.h[0]
+ SMLAL v18.4s, v4.4h, v2.h[0]
+ SMLAL2 v22.4s, v4.8h, v2.h[0]
+ SMLAL v26.4s, v5.4h, v2.h[0]
+ SMLAL2 v30.4s, v5.8h, v2.h[0]
+ SMLAL v19.4s, v4.4h, v3.h[0]
+ SMLAL2 v23.4s, v4.8h, v3.h[0]
+ SMLAL v27.4s, v5.4h, v3.h[0]
+ SMLAL2 v31.4s, v5.8h, v3.h[0]
+ CMP x0, 2
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[1]
+ SMLAL2 v20.4s, v4.8h, v0.h[1]
+ SMLAL v24.4s, v5.4h, v0.h[1]
+ SMLAL2 v28.4s, v5.8h, v0.h[1]
+ SMLAL v17.4s, v4.4h, v1.h[1]
+ SMLAL2 v21.4s, v4.8h, v1.h[1]
+ SMLAL v25.4s, v5.4h, v1.h[1]
+ SMLAL2 v29.4s, v5.8h, v1.h[1]
+ SMLAL v18.4s, v4.4h, v2.h[1]
+ SMLAL2 v22.4s, v4.8h, v2.h[1]
+ SMLAL v26.4s, v5.4h, v2.h[1]
+ SMLAL2 v30.4s, v5.8h, v2.h[1]
+ SMLAL v19.4s, v4.4h, v3.h[1]
+ SMLAL2 v23.4s, v4.8h, v3.h[1]
+ SMLAL v27.4s, v5.4h, v3.h[1]
+ SMLAL2 v31.4s, v5.8h, v3.h[1]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[2]
+ SMLAL2 v20.4s, v4.8h, v0.h[2]
+ SMLAL v24.4s, v5.4h, v0.h[2]
+ SMLAL2 v28.4s, v5.8h, v0.h[2]
+ SMLAL v17.4s, v4.4h, v1.h[2]
+ SMLAL2 v21.4s, v4.8h, v1.h[2]
+ SMLAL v25.4s, v5.4h, v1.h[2]
+ SMLAL2 v29.4s, v5.8h, v1.h[2]
+ SMLAL v18.4s, v4.4h, v2.h[2]
+ SMLAL2 v22.4s, v4.8h, v2.h[2]
+ SMLAL v26.4s, v5.4h, v2.h[2]
+ SMLAL2 v30.4s, v5.8h, v2.h[2]
+ SMLAL v19.4s, v4.4h, v3.h[2]
+ SMLAL2 v23.4s, v4.8h, v3.h[2]
+ SMLAL v27.4s, v5.4h, v3.h[2]
+ SMLAL2 v31.4s, v5.8h, v3.h[2]
+ CMP x0, 4
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[3]
+ SMLAL2 v20.4s, v4.8h, v0.h[3]
+ SMLAL v24.4s, v5.4h, v0.h[3]
+ SMLAL2 v28.4s, v5.8h, v0.h[3]
+ SMLAL v17.4s, v4.4h, v1.h[3]
+ SMLAL2 v21.4s, v4.8h, v1.h[3]
+ SMLAL v25.4s, v5.4h, v1.h[3]
+ SMLAL2 v29.4s, v5.8h, v1.h[3]
+ SMLAL v18.4s, v4.4h, v2.h[3]
+ SMLAL2 v22.4s, v4.8h, v2.h[3]
+ SMLAL v26.4s, v5.4h, v2.h[3]
+ SMLAL2 v30.4s, v5.8h, v2.h[3]
+ SMLAL v19.4s, v4.4h, v3.h[3]
+ SMLAL2 v23.4s, v4.8h, v3.h[3]
+ SMLAL v27.4s, v5.4h, v3.h[3]
+ SMLAL2 v31.4s, v5.8h, v3.h[3]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[4]
+ SMLAL2 v20.4s, v4.8h, v0.h[4]
+ SMLAL v24.4s, v5.4h, v0.h[4]
+ SMLAL2 v28.4s, v5.8h, v0.h[4]
+ SMLAL v17.4s, v4.4h, v1.h[4]
+ SMLAL2 v21.4s, v4.8h, v1.h[4]
+ SMLAL v25.4s, v5.4h, v1.h[4]
+ SMLAL2 v29.4s, v5.8h, v1.h[4]
+ SMLAL v18.4s, v4.4h, v2.h[4]
+ SMLAL2 v22.4s, v4.8h, v2.h[4]
+ SMLAL v26.4s, v5.4h, v2.h[4]
+ SMLAL2 v30.4s, v5.8h, v2.h[4]
+ SMLAL v19.4s, v4.4h, v3.h[4]
+ SMLAL2 v23.4s, v4.8h, v3.h[4]
+ SMLAL v27.4s, v5.4h, v3.h[4]
+ SMLAL2 v31.4s, v5.8h, v3.h[4]
+ CMP x0, 6
+ B.LO 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[5]
+ SMLAL2 v20.4s, v4.8h, v0.h[5]
+ SMLAL v24.4s, v5.4h, v0.h[5]
+ SMLAL2 v28.4s, v5.8h, v0.h[5]
+ SMLAL v17.4s, v4.4h, v1.h[5]
+ SMLAL2 v21.4s, v4.8h, v1.h[5]
+ SMLAL v25.4s, v5.4h, v1.h[5]
+ SMLAL2 v29.4s, v5.8h, v1.h[5]
+ SMLAL v18.4s, v4.4h, v2.h[5]
+ SMLAL2 v22.4s, v4.8h, v2.h[5]
+ SMLAL v26.4s, v5.4h, v2.h[5]
+ SMLAL2 v30.4s, v5.8h, v2.h[5]
+ SMLAL v19.4s, v4.4h, v3.h[5]
+ SMLAL2 v23.4s, v4.8h, v3.h[5]
+ SMLAL v27.4s, v5.4h, v3.h[5]
+ SMLAL2 v31.4s, v5.8h, v3.h[5]
+ B.EQ 4b
+
+ LDP d4, d5, [x5], 16
+ SXTL v4.8h, v4.8b
+ SXTL v5.8h, v5.8b
+ SMLAL v16.4s, v4.4h, v0.h[6]
+ SMLAL2 v20.4s, v4.8h, v0.h[6]
+ SMLAL v24.4s, v5.4h, v0.h[6]
+ SMLAL2 v28.4s, v5.8h, v0.h[6]
+ SMLAL v17.4s, v4.4h, v1.h[6]
+ SMLAL2 v21.4s, v4.8h, v1.h[6]
+ SMLAL v25.4s, v5.4h, v1.h[6]
+ SMLAL2 v29.4s, v5.8h, v1.h[6]
+ SMLAL v18.4s, v4.4h, v2.h[6]
+ SMLAL2 v22.4s, v4.8h, v2.h[6]
+ SMLAL v26.4s, v5.4h, v2.h[6]
+ SMLAL2 v30.4s, v5.8h, v2.h[6]
+ SMLAL v19.4s, v4.4h, v3.h[6]
+ SMLAL2 v23.4s, v4.8h, v3.h[6]
+ SMLAL v27.4s, v5.4h, v3.h[6]
+ SMLAL2 v31.4s, v5.8h, v3.h[6]
+ B 4b
+
+ # Store odd width
+ .p2align 3
+6:
+ TBZ x1, 3, 7f
+ STR d7, [x7], 8
+ DUP d7, v7.d[1]
+ STR d6, [x17], 8
+ DUP d6, v6.d[1]
+ STR d5, [x16], 8
+ DUP d5, v5.d[1]
+ STR d4, [x6], 8
+ DUP d4, v4.d[1]
+7:
+ TBZ x1, 2, 8f
+ STR s7, [x7], 4
+ DUP s7, v7.s[1]
+ STR s6, [x17], 4
+ DUP s6, v6.s[1]
+ STR s5, [x16], 4
+ DUP s5, v5.s[1]
+ STR s4, [x6], 4
+ DUP s4, v4.s[1]
+8:
+ TBZ x1, 1, 9f
+ ST1 {v7.h}[0], [x7], 2
+ DUP h7, v7.h[1]
+ ST1 {v6.h}[0], [x17], 2
+ DUP h6, v6.h[1]
+ ST1 {v5.h}[0], [x16], 2
+ DUP h5, v5.h[1]
+ ST1 {v4.h}[0], [x6], 2
+ DUP h4, v4.h[1]
+9:
+ TBZ x1, 0, 10f
+ ST1 {v7.b}[0], [x7]
+ ST1 {v6.b}[0], [x17]
+ ST1 {v5.b}[0], [x16]
+ ST1 {v4.b}[0], [x6]
+10:
+ # Restore x20-x21 from stack
+ LDP x20, x21, [sp], 16
+ RET
+
+END_FUNCTION xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+
+#ifdef __ELF__
+.section ".note.GNU-stack","",%progbits
+#endif
diff --git a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
index d4cf6a1..5ab1be4 100644
--- a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
+++ b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-cortex-a53.S
@@ -7,6 +7,7 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+
#include <xnnpack/assembly.h>
# void xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53(
diff --git a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
index ab656d4..5ab8061 100644
--- a/src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
+++ b/src/qs8-igemm/gen/4x16-minmax-gemmlowp-aarch64-neon-mlal-lane-prfm-cortex-a53.S
@@ -7,6 +7,7 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
+
#include <xnnpack/assembly.h>
# void xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53(
diff --git a/src/xnnpack/gemm.h b/src/xnnpack/gemm.h
index 34885c1..a05434a 100644
--- a/src/xnnpack/gemm.h
+++ b/src/xnnpack/gemm.h
@@ -648,6 +648,9 @@
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53)
+DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53)
+
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c4__aarch64_neondot_ld32)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c4__aarch64_neondot_ld64)
DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16c4__aarch64_neondot_ld32)
@@ -981,6 +984,9 @@
DECLARE_QC8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_gemm_minmax_fp32_ukernel_6x16c4__neondot)
DECLARE_QC8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_gemm_minmax_fp32_ukernel_8x16c4__neondot)
+DECLARE_QC8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53)
+DECLARE_QC8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53)
+
DECLARE_QC8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_gemm_minmax_fp32_ukernel_1x4c2__sse2_ld64)
DECLARE_QC8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_gemm_minmax_fp32_ukernel_2x4c2__sse2_ld64)
DECLARE_QC8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_gemm_minmax_fp32_ukernel_3x4c2__sse2_ld64)
diff --git a/src/xnnpack/igemm.h b/src/xnnpack/igemm.h
index 592b36b..350d15e 100644
--- a/src/xnnpack/igemm.h
+++ b/src/xnnpack/igemm.h
@@ -458,6 +458,9 @@
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53)
+DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53)
+
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c4__aarch64_neondot_ld64)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c4__aarch64_neondot_ld128)
DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c4__aarch64_neondot_cortex_a55)
@@ -730,6 +733,9 @@
DECLARE_QC8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_igemm_minmax_fp32_ukernel_6x16c4__neondot)
DECLARE_QC8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_igemm_minmax_fp32_ukernel_8x16c4__neondot)
+DECLARE_QC8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53)
+DECLARE_QC8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53)
+
DECLARE_QC8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_igemm_minmax_fp32_ukernel_1x4c2__sse2_ld64)
DECLARE_QC8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_igemm_minmax_fp32_ukernel_2x4c2__sse2_ld64)
DECLARE_QC8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qc8_igemm_minmax_fp32_ukernel_3x4c2__sse2_ld64)
diff --git a/test/qc8-gemm-minmax-fp32.cc b/test/qc8-gemm-minmax-fp32.cc
index 9d70f20..9b2dff8 100644
--- a/test/qc8-gemm-minmax-fp32.cc
+++ b/test/qc8-gemm-minmax-fp32.cc
@@ -22,6 +22,918 @@
#include "gemm-microkernel-tester.h"
+#if XNN_ARCH_ARM64
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_ARM64
+
+
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
TEST(QC8_GEMM_MINMAX_FP32_1X16__NEON_MLAL_LANE, k_eq_8) {
TEST_REQUIRES_ARM_NEON;
diff --git a/test/qc8-gemm-minmax-fp32.yaml b/test/qc8-gemm-minmax-fp32.yaml
index 8dc3a67..4fc9ce8 100644
--- a/test/qc8-gemm-minmax-fp32.yaml
+++ b/test/qc8-gemm-minmax-fp32.yaml
@@ -3,6 +3,12 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
+- name: xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+ init: xnn_init_qs8_minmax_neon_params
+ k-block: 8
+- name: xnn_qc8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+ init: xnn_init_qs8_minmax_neon_params
+ k-block: 8
- name: xnn_qc8_gemm_minmax_fp32_ukernel_1x16__neon_mlal_lane
init: xnn_init_qs8_minmax_neon_fp32_params
k-block: 8
diff --git a/test/qc8-igemm-minmax-fp32.cc b/test/qc8-igemm-minmax-fp32.cc
index 26d37f8..4bcaeef 100644
--- a/test/qc8-igemm-minmax-fp32.cc
+++ b/test/qc8-igemm-minmax-fp32.cc
@@ -22,6 +22,942 @@
#include "gemm-microkernel-tester.h"
+#if XNN_ARCH_ARM64
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, small_kernel_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, a_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 4; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .zero_index(mz)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, small_kernel_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, a_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 4; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .zero_index(mz)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_ARM64
+
+
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
TEST(QC8_IGEMM_MINMAX_FP32_1X16__NEON_MLAL_LANE, k_eq_8) {
TEST_REQUIRES_ARM_NEON;
diff --git a/test/qc8-igemm-minmax-fp32.yaml b/test/qc8-igemm-minmax-fp32.yaml
index 6b8a005..5c92258 100644
--- a/test/qc8-igemm-minmax-fp32.yaml
+++ b/test/qc8-igemm-minmax-fp32.yaml
@@ -3,6 +3,12 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
+- name: xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+ init: xnn_init_qs8_minmax_neon_params
+ k-block: 8
+- name: xnn_qc8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+ init: xnn_init_qs8_minmax_neon_params
+ k-block: 8
- name: xnn_qc8_igemm_minmax_fp32_ukernel_1x16__neon_mlal_lane
init: xnn_init_qs8_minmax_neon_fp32_params
k-block: 8
diff --git a/test/qs8-gemm-minmax-fp32.cc b/test/qs8-gemm-minmax-fp32.cc
index 36a154b..c80a998 100644
--- a/test/qs8-gemm-minmax-fp32.cc
+++ b/test/qs8-gemm-minmax-fp32.cc
@@ -23,6 +23,918 @@
#if XNN_ARCH_ARM64
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_GEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
TEST(QS8_GEMM_MINMAX_FP32_1X16C4__AARCH64_NEONDOT_LD32, k_eq_4) {
TEST_REQUIRES_ARM_NEON_DOT;
GemmMicrokernelTester()
diff --git a/test/qs8-gemm-minmax-fp32.yaml b/test/qs8-gemm-minmax-fp32.yaml
index 37cbbb0..f66725b 100644
--- a/test/qs8-gemm-minmax-fp32.yaml
+++ b/test/qs8-gemm-minmax-fp32.yaml
@@ -3,6 +3,12 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+ init: xnn_init_qs8_conv_minmax_fp32_neonv8_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+ init: xnn_init_qs8_conv_minmax_fp32_neonv8_params
+ k-block: 8
- name: xnn_qs8_gemm_minmax_fp32_ukernel_1x16c4__aarch64_neondot_ld32
init: xnn_init_qs8_conv_minmax_fp32_neonv8_params
k-block: 4
diff --git a/test/qs8-gemm-minmax-gemmlowp.cc b/test/qs8-gemm-minmax-gemmlowp.cc
index bf6466b..709167f 100644
--- a/test/qs8-gemm-minmax-gemmlowp.cc
+++ b/test/qs8-gemm-minmax-gemmlowp.cc
@@ -23,6 +23,918 @@
#if XNN_ARCH_ARM64
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(11)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .a_stride(83)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_a) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .a_stride(43)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
TEST(QS8_GEMM_MINMAX_GEMMLOWP_1X16C4__AARCH64_NEONDOT_LD32, k_eq_4) {
TEST_REQUIRES_ARM_NEON_DOT;
GemmMicrokernelTester()
@@ -2759,918 +3671,6 @@
#if XNN_ARCH_ARM64
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .cn_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .a_stride(11)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_m) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(16)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_n) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .a_stride(11)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .a_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .a_stride(83)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .cn_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(k)
- .a_stride(43)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(k)
- .cn_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(k)
- .a_stride(43)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .cm_stride(19)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmin) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .qmin(128)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmax) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .qmax(128)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .cm_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-#endif // XNN_ARCH_ARM64
-
-
-#if XNN_ARCH_ARM64
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .cn_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .a_stride(11)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_m) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(16)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_n) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .a_stride(11)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .a_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .a_stride(83)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .cn_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(k)
- .a_stride(43)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(k)
- .cn_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_a) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(k)
- .a_stride(43)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .cm_stride(19)
- .iterations(1)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmin) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .qmin(128)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmax) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .qmax(128)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_GEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .cm_stride(19)
- .Test(xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-#endif // XNN_ARCH_ARM64
-
-
-#if XNN_ARCH_ARM64
TEST(QS8_GEMM_MINMAX_GEMMLOWP_1X8C8__AARCH64_NEON_MLAL_PADAL, k_eq_16) {
TEST_REQUIRES_ARM_NEON;
GemmMicrokernelTester()
diff --git a/test/qs8-gemm-minmax-gemmlowp.yaml b/test/qs8-gemm-minmax-gemmlowp.yaml
index 878e34a..9748274 100644
--- a/test/qs8-gemm-minmax-gemmlowp.yaml
+++ b/test/qs8-gemm-minmax-gemmlowp.yaml
@@ -3,6 +3,12 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
+- name: xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+ init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
+ k-block: 8
+- name: xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+ init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
+ k-block: 8
- name: xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x16c4__aarch64_neondot_ld32
init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
k-block: 4
@@ -21,12 +27,6 @@
- name: xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16c4__aarch64_neondot_cortex_a55
init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
k-block: 16
-- name: xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
- init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
- k-block: 8
-- name: xnn_qs8_gemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
- init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
- k-block: 8
- name: xnn_qs8_gemm_minmax_gemmlowp_ukernel_1x8c8__aarch64_neon_mlal_padal
init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
k-block: 16
diff --git a/test/qs8-igemm-minmax-fp32.cc b/test/qs8-igemm-minmax-fp32.cc
index 4db5f4e..6ce0d06 100644
--- a/test/qs8-igemm-minmax-fp32.cc
+++ b/test/qs8-igemm-minmax-fp32.cc
@@ -23,6 +23,942 @@
#if XNN_ARCH_ARM64
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, small_kernel_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, a_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 4; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .zero_index(mz)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, small_kernel_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, a_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 4; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .zero_index(mz)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_FP32_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_fp32_neonv8_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
TEST(QS8_IGEMM_MINMAX_FP32_4X16C4__AARCH64_NEONDOT_CORTEX_A55, k_eq_16) {
TEST_REQUIRES_ARM_NEON_DOT;
GemmMicrokernelTester()
diff --git a/test/qs8-igemm-minmax-fp32.yaml b/test/qs8-igemm-minmax-fp32.yaml
index 7861808..85e0740 100644
--- a/test/qs8-igemm-minmax-fp32.yaml
+++ b/test/qs8-igemm-minmax-fp32.yaml
@@ -3,6 +3,12 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
+- name: xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+ init: xnn_init_qs8_conv_minmax_fp32_neonv8_params
+ k-block: 8
+- name: xnn_qs8_igemm_minmax_fp32_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+ init: xnn_init_qs8_conv_minmax_fp32_neonv8_params
+ k-block: 8
- name: xnn_qs8_igemm_minmax_fp32_ukernel_4x16c4__aarch64_neondot_cortex_a55
init: xnn_init_qs8_conv_minmax_fp32_neonv8_params
k-block: 16
diff --git a/test/qs8-igemm-minmax-gemmlowp.cc b/test/qs8-igemm-minmax-gemmlowp.cc
index fb8862e..6ab8d9c 100644
--- a/test/qs8-igemm-minmax-gemmlowp.cc
+++ b/test/qs8-igemm-minmax-gemmlowp.cc
@@ -23,6 +23,942 @@
#if XNN_ARCH_ARM64
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, small_kernel_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, a_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 4; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .zero_index(mz)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_m) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(16)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_n) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(8)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k < 8; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 9; k < 16; k++) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 16; k <= 80; k += 8) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_cn) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(n)
+ .k(k)
+ .cn_stride(19)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, small_kernel_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .ks(3)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 17; n < 32; n++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_small_kernel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t n = 32; n <= 48; n += 16) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm_subtile) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ for (uint32_t m = 1; m <= 4; m++) {
+ for (uint32_t n = 1; n <= 16; n++) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(m)
+ .n(n)
+ .k(k)
+ .cm_stride(19)
+ .iterations(1)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, a_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 4; mz++) {
+ for (size_t k = 1; k <= 40; k += 9) {
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(k)
+ .ks(3)
+ .a_offset(163)
+ .zero_index(mz)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+ }
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmin(128)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .qmax(128)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+
+ TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm) {
+ TEST_REQUIRES_ARM_NEON;
+ GemmMicrokernelTester()
+ .mr(4)
+ .nr(16)
+ .kr(1)
+ .sr(1)
+ .m(4)
+ .n(16)
+ .k(8)
+ .cm_stride(19)
+ .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
+ }
+#endif // XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM64
TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16C4__AARCH64_NEONDOT_CORTEX_A55, k_eq_16) {
TEST_REQUIRES_ARM_NEON_DOT;
GemmMicrokernelTester()
@@ -1427,942 +2363,6 @@
#if XNN_ARCH_ARM64
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .cn_stride(19)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_m) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(16)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_eq_8_subtile_n) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_lt_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_gt_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, k_div_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .cn_stride(19)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(k)
- .cn_stride(19)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, small_kernel) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, small_kernel_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .ks(3)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_gt_16_small_kernel) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, n_div_16_small_kernel) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .cm_stride(19)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, a_offset) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .a_offset(163)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, zero) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t mz = 0; mz < 4; mz++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .a_offset(163)
- .zero_index(mz)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmin) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .qmin(128)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, qmax) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .qmax(128)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_PRFM_CORTEX_A53, strided_cm) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .cm_stride(19)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-#endif // XNN_ARCH_ARM64
-
-
-#if XNN_ARCH_ARM64
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .cn_stride(19)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_m) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(16)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_eq_8_subtile_n) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(8)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_lt_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k < 8; k++) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_gt_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 9; k < 16; k++) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, k_div_8_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 16; k <= 80; k += 8) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .cn_stride(19)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_strided_cn) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(n)
- .k(k)
- .cn_stride(19)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, small_kernel) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, small_kernel_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .ks(3)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_gt_16_small_kernel) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 17; n < 32; n++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, n_div_16_small_kernel) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t n = 32; n <= 48; n += 16) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm_subtile) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- for (uint32_t m = 1; m <= 4; m++) {
- for (uint32_t n = 1; n <= 16; n++) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(m)
- .n(n)
- .k(k)
- .cm_stride(19)
- .iterations(1)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, a_offset) {
- TEST_REQUIRES_ARM_NEON;
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .a_offset(163)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, zero) {
- TEST_REQUIRES_ARM_NEON;
- for (uint32_t mz = 0; mz < 4; mz++) {
- for (size_t k = 1; k <= 40; k += 9) {
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(k)
- .ks(3)
- .a_offset(163)
- .zero_index(mz)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
- }
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmin) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .qmin(128)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, qmax) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .qmax(128)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-
- TEST(QS8_IGEMM_MINMAX_GEMMLOWP_4X16__AARCH64_NEON_MLAL_LANE_CORTEX_A53, strided_cm) {
- TEST_REQUIRES_ARM_NEON;
- GemmMicrokernelTester()
- .mr(4)
- .nr(16)
- .kr(1)
- .sr(1)
- .m(4)
- .n(16)
- .k(8)
- .cm_stride(19)
- .Test(xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53, xnn_init_qs8_conv_minmax_gemmlowp_neon_params, xnn_init_qs8_requantization_gemmlowp_params, xnn_qs8_requantize_gemmlowp);
- }
-#endif // XNN_ARCH_ARM64
-
-
-#if XNN_ARCH_ARM64
TEST(QS8_IGEMM_MINMAX_GEMMLOWP_1X8C8__AARCH64_NEON_MLAL_PADAL, k_eq_16) {
TEST_REQUIRES_ARM_NEON;
GemmMicrokernelTester()
diff --git a/test/qs8-igemm-minmax-gemmlowp.yaml b/test/qs8-igemm-minmax-gemmlowp.yaml
index 348ae0e..5cc6a7d 100644
--- a/test/qs8-igemm-minmax-gemmlowp.yaml
+++ b/test/qs8-igemm-minmax-gemmlowp.yaml
@@ -3,6 +3,12 @@
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
+- name: xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
+ init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
+ k-block: 8
+- name: xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
+ init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
+ k-block: 8
- name: xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c4__aarch64_neondot_cortex_a55
init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
k-block: 16
@@ -12,12 +18,6 @@
- name: xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16c4__aarch64_neondot_ld128
init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
k-block: 16
-- name: xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_prfm_cortex_a53
- init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
- k-block: 8
-- name: xnn_qs8_igemm_minmax_gemmlowp_ukernel_4x16__aarch64_neon_mlal_lane_cortex_a53
- init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
- k-block: 8
- name: xnn_qs8_igemm_minmax_gemmlowp_ukernel_1x8c8__aarch64_neon_mlal_padal
init: xnn_init_qs8_conv_minmax_gemmlowp_neon_params
k-block: 16
