Accumulate in 16 bits once in NEON QS8/QC8 DWCONV before extending to 32 bits
Re-use the trick from QS8/QC8 GEMM microkernels for DWCONV.
PiperOrigin-RevId: 386393857
diff --git a/BUILD.bazel b/BUILD.bazel
index 82d2145..27ab07c 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -1929,12 +1929,16 @@
"src/math/sqrt-neon-nr1rsqrts.c",
"src/math/sqrt-neon-nr2rsqrts.c",
"src/math/sqrt-neon-nr3rsqrts.c",
+ "src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mla8.c",
"src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul8.c",
"src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c",
+ "src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mla8.c",
"src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul8.c",
"src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul16.c",
+ "src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mla8.c",
"src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul8.c",
"src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c",
+ "src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mla8.c",
"src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul8.c",
"src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul16.c",
"src/qc8-dwconv/gen/up24x9-minmax-fp32-neon-mul16.c",
@@ -1955,18 +1959,22 @@
"src/qc8-igemm/gen/4x16-minmax-fp32-neon-mlal-lane.c",
"src/qs8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c",
"src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-neon-mul16.c",
+ "src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mla8.c",
"src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul8.c",
"src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul16.c",
"src/qs8-dwconv/gen/up8x25-minmax-fp32-neon-mul16.c",
"src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-neon-mul16.c",
+ "src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mla8.c",
"src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c",
"src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul16.c",
"src/qs8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c",
"src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-neon-mul16.c",
+ "src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mla8.c",
"src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul8.c",
"src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul16.c",
"src/qs8-dwconv/gen/up16x25-minmax-fp32-neon-mul16.c",
"src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-neon-mul16.c",
+ "src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mla8.c",
"src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul8.c",
"src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul16.c",
"src/qs8-dwconv/gen/up24x9-minmax-fp32-neon-mul16.c",
@@ -2533,12 +2541,16 @@
"src/math/roundne-neonv8.c",
"src/math/roundu-neonv8.c",
"src/math/roundz-neonv8.c",
+ "src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mla8.c",
"src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul8.c",
"src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul16.c",
+ "src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mla8.c",
"src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul8.c",
"src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul16.c",
+ "src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mla8.c",
"src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul8.c",
"src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul16.c",
+ "src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mla8.c",
"src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul8.c",
"src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul16.c",
"src/qc8-dwconv/gen/up24x9-minmax-fp32-neonv8-mul16.c",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a761c40..73d90ac 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1118,12 +1118,16 @@
src/math/sqrt-neon-nr1rsqrts.c
src/math/sqrt-neon-nr2rsqrts.c
src/math/sqrt-neon-nr3rsqrts.c
+ src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mla8.c
src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul8.c
src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c
+ src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mla8.c
src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul8.c
src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul16.c
+ src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mla8.c
src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul8.c
src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c
+ src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mla8.c
src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul8.c
src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul16.c
src/qc8-dwconv/gen/up24x9-minmax-fp32-neon-mul16.c
@@ -1144,18 +1148,22 @@
src/qc8-igemm/gen/4x16-minmax-fp32-neon-mlal-lane.c
src/qs8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c
src/qs8-dwconv/gen/up8x9-minmax-gemmlowp-neon-mul16.c
+ src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mla8.c
src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul8.c
src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul16.c
src/qs8-dwconv/gen/up8x25-minmax-fp32-neon-mul16.c
src/qs8-dwconv/gen/up8x25-minmax-gemmlowp-neon-mul16.c
+ src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mla8.c
src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c
src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul16.c
src/qs8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c
src/qs8-dwconv/gen/up16x9-minmax-gemmlowp-neon-mul16.c
+ src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mla8.c
src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul8.c
src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul16.c
src/qs8-dwconv/gen/up16x25-minmax-fp32-neon-mul16.c
src/qs8-dwconv/gen/up16x25-minmax-gemmlowp-neon-mul16.c
+ src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mla8.c
src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul8.c
src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul16.c
src/qs8-dwconv/gen/up24x9-minmax-fp32-neon-mul16.c
@@ -1605,9 +1613,17 @@
src/math/roundne-neonv8.c
src/math/roundu-neonv8.c
src/math/roundz-neonv8.c
+ src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mla8.c
+ src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul8.c
src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul16.c
+ src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mla8.c
+ src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul8.c
src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul16.c
+ src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mla8.c
+ src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul8.c
src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul16.c
+ src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mla8.c
+ src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul8.c
src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul16.c
src/qc8-dwconv/gen/up24x9-minmax-fp32-neonv8-mul16.c
src/qc8-dwconv/gen/up24x25-minmax-fp32-neonv8-mul16.c
@@ -1626,9 +1642,17 @@
src/qc8-igemm/gen/2x8c8-minmax-fp32-neonv8-mlal-padal.c
src/qc8-igemm/gen/4x16-minmax-fp32-neonv8-mlal-lane.c
src/qs8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul16.c
+ src/qs8-dwconv/gen/up8x9-minmax-rndnu-neonv8-mla8.c
+ src/qs8-dwconv/gen/up8x9-minmax-rndnu-neonv8-mul8.c
src/qs8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul16.c
+ src/qs8-dwconv/gen/up8x25-minmax-rndnu-neonv8-mla8.c
+ src/qs8-dwconv/gen/up8x25-minmax-rndnu-neonv8-mul8.c
src/qs8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul16.c
+ src/qs8-dwconv/gen/up16x9-minmax-rndnu-neonv8-mla8.c
+ src/qs8-dwconv/gen/up16x9-minmax-rndnu-neonv8-mul8.c
src/qs8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul16.c
+ src/qs8-dwconv/gen/up16x25-minmax-rndnu-neonv8-mla8.c
+ src/qs8-dwconv/gen/up16x25-minmax-rndnu-neonv8-mul8.c
src/qs8-dwconv/gen/up24x9-minmax-fp32-neonv8-mul16.c
src/qs8-dwconv/gen/up24x25-minmax-fp32-neonv8-mul16.c
src/qs8-dwconv/gen/up32x9-minmax-fp32-neonv8-mul16.c
diff --git a/scripts/generate-qs8-dwconv.sh b/scripts/generate-qs8-dwconv.sh
index 01d3025..ebd2261 100755
--- a/scripts/generate-qs8-dwconv.sh
+++ b/scripts/generate-qs8-dwconv.sh
@@ -62,23 +62,41 @@
tools/xngen src/qs8-dwconv/unipass-scalar.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D VARIANT=MAGIC -D DATATYPE=QU8 -o src/qu8-dwconv/gen/up4x25-minmax-fp32-scalar-magic.c
################################## ARM NEON ##################################
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=0 -o src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=0 -o src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=0 -D ARMV8=0 -o src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=0 -D ARMV8=0 -o src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=1 -o src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=1 -o src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=1 -D ARMV8=0 -o src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mla8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=1 -D ARMV8=0 -o src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mla8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D ARMV8=0 -o src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D ARMV8=0 -o src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=0 -D ARMV8=1 -o src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=0 -D ARMV8=1 -o src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=0 -o src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=0 -o src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=1 -D ARMV8=1 -o src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mla8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=1 -D ARMV8=1 -o src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mla8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=1 -o src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=1 -o src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D MLA=0 -D ARMV8=0 -o src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D MLA=0 -D ARMV8=0 -o src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D ARMV8=0 -o src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c
-tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D ARMV8=0 -o src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D MLA=1 -D ARMV8=0 -o src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mla8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D MLA=1 -D ARMV8=0 -o src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mla8.c
+
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=0 -D ARMV8=0 -o src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=0 -D ARMV8=0 -o src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul8.c
+
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=1 -D ARMV8=0 -o src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mla8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=1 -D ARMV8=0 -o src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mla8.c
+
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=0 -D ARMV8=1 -o src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=0 -D ARMV8=1 -o src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul8.c
+
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=1 -D ARMV8=1 -o src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mla8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D MLA=1 -D ARMV8=1 -o src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mla8.c
+
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D MLA=0 -D ARMV8=0 -o src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D MLA=0 -D ARMV8=0 -o src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul8.c
+
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D MLA=1 -D ARMV8=0 -o src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mla8.c
+tools/xngen src/qs8-dwconv/unipass-neon-mul8.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -D REQUANTIZATION=RNDNU -D DATATYPE=QS8 -D MLA=1 -D ARMV8=0 -o src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mla8.c
tools/xngen src/qs8-dwconv/unipass-neon-mul16.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=0 -o src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul16.c
tools/xngen src/qs8-dwconv/unipass-neon-mul16.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=9 -D REQUANTIZATION=FP32 -D DATATYPE=QC8 -D ARMV8=0 -o src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul16.c
diff --git a/src/init.c b/src/init.c
index 5b20a93..6965797 100644
--- a/src/init.c
+++ b/src/init.c
@@ -129,11 +129,11 @@
xnn_params.qc8.gemm.log2_kr = 1;
}
- xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mul8;
+ xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8;
xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_neon_fp32_params;
xnn_params.qc8.dwconv[0].channel_tile = 8;
xnn_params.qc8.dwconv[0].primary_tile = 9;
- xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mul8;
+ xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8;
xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_neon_fp32_params;
xnn_params.qc8.dwconv[1].channel_tile = 8;
xnn_params.qc8.dwconv[1].primary_tile = 25;
@@ -163,11 +163,11 @@
xnn_params.qs8.gemm.log2_kr = 1;
}
- xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mul8;
+ xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8;
xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[0].primary_tile = 9;
- xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mul8;
+ xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8;
xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
xnn_params.qs8.dwconv[1].primary_tile = 25;
@@ -1063,11 +1063,11 @@
#endif // XNN_ENABLE_ASSEMBLY
#endif // XNN_PLATFORM_IOS || XNN_PLATFORM_MAC
- xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mul8;
+ xnn_params.qc8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8;
xnn_params.qc8.dwconv[0].init.qc8 = xnn_init_qs8_minmax_neon_params;
xnn_params.qc8.dwconv[0].channel_tile = 8;
xnn_params.qc8.dwconv[0].primary_tile = 9;
- xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mul8;
+ xnn_params.qc8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8;
xnn_params.qc8.dwconv[1].init.qc8 = xnn_init_qs8_minmax_neon_params;
xnn_params.qc8.dwconv[1].channel_tile = 8;
xnn_params.qc8.dwconv[1].primary_tile = 25;
@@ -1242,11 +1242,11 @@
#endif // XNN_ENABLE_ASSEMBLY
#endif // XNN_PLATFORM_IOS || XNN_PLATFORM_MAC
- xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mul8;
+ xnn_params.qs8.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8;
xnn_params.qs8.dwconv[0].init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
xnn_params.qs8.dwconv[0].channel_tile = 8;
xnn_params.qs8.dwconv[0].primary_tile = 9;
- xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mul8;
+ xnn_params.qs8.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8;
xnn_params.qs8.dwconv[1].init.qs8 = xnn_init_qs8_conv_minmax_rndnu_neon_params;
xnn_params.qs8.dwconv[1].channel_tile = 8;
xnn_params.qs8.dwconv[1].primary_tile = 25;
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mla8.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mla8.c
new file mode 100644
index 0000000..29f06ce
--- /dev/null
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mla8.c
@@ -0,0 +1,709 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const float32x4_t voutput_min_less_zero_point = vld1q_dup_f32(¶ms->neon_fp32.output_min_less_zero_point);
+ const float32x4_t voutput_max_less_zero_point = vld1q_dup_f32(¶ms->neon_fp32.output_max_less_zero_point);
+ const float32x4_t vmagic_bias = vld1q_dup_f32(¶ms->neon_fp32.magic_bias);
+ const int32x4_t vmagic_bias_less_zero_point = vld1q_dup_s32(¶ms->neon_fp32.magic_bias_less_zero_point);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi9x89ABCDEF = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi9x89ABCDEF, vk9x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi10x89ABCDEF = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
+
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi11x89ABCDEF = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi11x89ABCDEF, vk11x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi12x89ABCDEF = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
+
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi13x89ABCDEF = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi13x89ABCDEF, vk13x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi14x89ABCDEF = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
+
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi15x89ABCDEF = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi15x89ABCDEF, vk15x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi16x89ABCDEF = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
+
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi17x89ABCDEF = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi17x89ABCDEF, vk17x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi18x89ABCDEF = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
+
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi19x89ABCDEF = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi19x89ABCDEF, vk19x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi20x89ABCDEF = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
+
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi21x89ABCDEF = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi21x89ABCDEF, vk21x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi22x89ABCDEF = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
+
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi23x89ABCDEF = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi23x89ABCDEF, vk23x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi24x89ABCDEF = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+ float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
+ float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale4567 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale89AB = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscaleCDEF = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+ vfpacc89AB = vmulq_f32(vfpacc89AB, vscale89AB);
+ vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscaleCDEF);
+
+ vfpacc0123 = vmaxq_f32(vfpacc0123, voutput_min_less_zero_point);
+ vfpacc4567 = vmaxq_f32(vfpacc4567, voutput_min_less_zero_point);
+ vfpacc89AB = vmaxq_f32(vfpacc89AB, voutput_min_less_zero_point);
+ vfpaccCDEF = vmaxq_f32(vfpaccCDEF, voutput_min_less_zero_point);
+
+ vfpacc0123 = vminq_f32(vfpacc0123, voutput_max_less_zero_point);
+ vfpacc4567 = vminq_f32(vfpacc4567, voutput_max_less_zero_point);
+ vfpacc89AB = vminq_f32(vfpacc89AB, voutput_max_less_zero_point);
+ vfpaccCDEF = vminq_f32(vfpaccCDEF, voutput_max_less_zero_point);
+
+ vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
+ vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
+ vacc89AB = vreinterpretq_s32_f32(vaddq_f32(vfpacc89AB, vmagic_bias));
+ vaccCDEF = vreinterpretq_s32_f32(vaddq_f32(vfpaccCDEF, vmagic_bias));
+
+ vacc0123 = vsubq_s32(vacc0123, vmagic_bias_less_zero_point);
+ vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
+ vacc89AB = vsubq_s32(vacc89AB, vmagic_bias_less_zero_point);
+ vaccCDEF = vsubq_s32(vaccCDEF, vmagic_bias_less_zero_point);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc89AB), vreinterpretq_s16_s32(vaccCDEF));
+
+ int8x16_t vout0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc01234567), vreinterpretq_s8_s16(vacc89ABCDEF));
+#else
+ const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vcombine_s16(vmovn_s32(vacc89AB), vmovn_s32(vaccCDEF));
+
+ int8x16_t vout0123456789ABCDEF = vcombine_s8(vmovn_s16(vacc01234567), vmovn_s16(vacc89ABCDEF));
+#endif
+
+
+ vst1q_s8(output, vout0123456789ABCDEF); output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
+ do {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8((const void*) (k + 136));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8((const void*) (k + 152));
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8((const void*) (k + 168));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8((const void*) (k + 184));
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8((const void*) (k + 200));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8((const void*) (k + 216));
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8((const void*) (k + 232));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8((const void*) (k + 248));
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8((const void*) (k + 264));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8((const void*) (k + 280));
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8((const void*) (k + 296));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8((const void*) (k + 312));
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8((const void*) (k + 328));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8((const void*) (k + 344));
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8((const void*) (k + 360));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8((const void*) (k + 376));
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) ((uintptr_t) w + 8 * sizeof(int32_t) + 400 * sizeof(int8_t)));
+ const float32x4_t vscale4567 = vld1q_f32((const float*) ((uintptr_t) w + 8 * sizeof(int32_t) + 400 * sizeof(int8_t) + 4 * sizeof(float)));
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vfpacc0123 = vmaxq_f32(vfpacc0123, voutput_min_less_zero_point);
+ vfpacc4567 = vmaxq_f32(vfpacc4567, voutput_min_less_zero_point);
+
+ vfpacc0123 = vminq_f32(vfpacc0123, voutput_max_less_zero_point);
+ vfpacc4567 = vminq_f32(vfpacc4567, voutput_max_less_zero_point);
+
+ vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
+ vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
+
+ vacc0123 = vsubq_s32(vacc0123, vmagic_bias_less_zero_point);
+ vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#endif
+
+
+ if XNN_LIKELY(c >= 8) {
+ vst1_s8(output, vout01234567); output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul8.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul8.c
index beb8e90..d567e83 100644
--- a/src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul8.c
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-neon-mul8.c
@@ -175,325 +175,325 @@
const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- const int16x8_t vprod0x89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod0x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod0x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- const int16x8_t vprod1x89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod1x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod1x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- const int16x8_t vprod2x89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod2x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod2x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- const int16x8_t vprod3x89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod3x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod3x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- const int16x8_t vprod4x89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod4x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod4x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- const int16x8_t vprod5x89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod5x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod5x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- const int16x8_t vprod6x89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod6x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod6x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- const int16x8_t vprod7x89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod7x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod7x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- const int16x8_t vprod8x89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod8x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod8x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi9x89ABCDEF = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- const int16x8_t vprod9x89ABCDEF = vmull_s8(vi9x89ABCDEF, vk9x89ABCDEF);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod89ABCDEF = vmull_s8(vi9x89ABCDEF, vk9x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod9x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod9x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi10x89ABCDEF = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- const int16x8_t vprod10x89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod10x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod10x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi11x89ABCDEF = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- const int16x8_t vprod11x89ABCDEF = vmull_s8(vi11x89ABCDEF, vk11x89ABCDEF);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod89ABCDEF = vmull_s8(vi11x89ABCDEF, vk11x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod11x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod11x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi12x89ABCDEF = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- const int16x8_t vprod12x89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod12x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod12x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi13x89ABCDEF = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- const int16x8_t vprod13x89ABCDEF = vmull_s8(vi13x89ABCDEF, vk13x89ABCDEF);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod89ABCDEF = vmull_s8(vi13x89ABCDEF, vk13x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod13x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod13x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi14x89ABCDEF = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- const int16x8_t vprod14x89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod14x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod14x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi15x89ABCDEF = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- const int16x8_t vprod15x89ABCDEF = vmull_s8(vi15x89ABCDEF, vk15x89ABCDEF);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod89ABCDEF = vmull_s8(vi15x89ABCDEF, vk15x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod15x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod15x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi16x89ABCDEF = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- const int16x8_t vprod16x89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod16x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod16x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi17x89ABCDEF = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- const int16x8_t vprod17x89ABCDEF = vmull_s8(vi17x89ABCDEF, vk17x89ABCDEF);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod89ABCDEF = vmull_s8(vi17x89ABCDEF, vk17x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod17x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod17x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi18x89ABCDEF = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- const int16x8_t vprod18x89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod18x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod18x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi19x89ABCDEF = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- const int16x8_t vprod19x89ABCDEF = vmull_s8(vi19x89ABCDEF, vk19x89ABCDEF);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod89ABCDEF = vmull_s8(vi19x89ABCDEF, vk19x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod19x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod19x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi20x89ABCDEF = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- const int16x8_t vprod20x89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod20x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod20x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi21x89ABCDEF = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- const int16x8_t vprod21x89ABCDEF = vmull_s8(vi21x89ABCDEF, vk21x89ABCDEF);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod89ABCDEF = vmull_s8(vi21x89ABCDEF, vk21x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod21x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod21x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi22x89ABCDEF = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- const int16x8_t vprod22x89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod22x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod22x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi23x89ABCDEF = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- const int16x8_t vprod23x89ABCDEF = vmull_s8(vi23x89ABCDEF, vk23x89ABCDEF);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod89ABCDEF = vmull_s8(vi23x89ABCDEF, vk23x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod23x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod23x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi24x89ABCDEF = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- const int16x8_t vprod24x89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod24x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod24x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
@@ -554,178 +554,178 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8((const void*) (k + 136));
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8((const void*) (k + 152));
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8((const void*) (k + 168));
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8((const void*) (k + 184));
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8((const void*) (k + 200));
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8((const void*) (k + 216));
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8((const void*) (k + 232));
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8((const void*) (k + 248));
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8((const void*) (k + 264));
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8((const void*) (k + 280));
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8((const void*) (k + 296));
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8((const void*) (k + 312));
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8((const void*) (k + 328));
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8((const void*) (k + 344));
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8((const void*) (k + 360));
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8((const void*) (k + 376));
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mla8.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mla8.c
new file mode 100644
index 0000000..0650ff8
--- /dev/null
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mla8.c
@@ -0,0 +1,690 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/intrinsics-polyfill.h>
+
+
+void xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi9x89ABCDEF = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi9x89ABCDEF, vk9x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi10x89ABCDEF = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
+
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi11x89ABCDEF = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi11x89ABCDEF, vk11x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi12x89ABCDEF = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
+
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi13x89ABCDEF = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi13x89ABCDEF, vk13x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi14x89ABCDEF = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
+
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi15x89ABCDEF = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi15x89ABCDEF, vk15x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi16x89ABCDEF = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
+
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi17x89ABCDEF = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi17x89ABCDEF, vk17x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi18x89ABCDEF = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
+
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi19x89ABCDEF = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi19x89ABCDEF, vk19x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi20x89ABCDEF = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
+
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi21x89ABCDEF = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi21x89ABCDEF, vk21x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi22x89ABCDEF = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
+
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi23x89ABCDEF = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi23x89ABCDEF, vk23x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi24x89ABCDEF = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+ float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
+ float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale4567 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale89AB = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscaleCDEF = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+ vfpacc89AB = vmulq_f32(vfpacc89AB, vscale89AB);
+ vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscaleCDEF);
+
+ vacc0123 = vcvtnq_s32_f32(vfpacc0123);
+ vacc4567 = vcvtnq_s32_f32(vfpacc4567);
+ vacc89AB = vcvtnq_s32_f32(vfpacc89AB);
+ vaccCDEF = vcvtnq_s32_f32(vfpaccCDEF);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ const int16x8_t vacc89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF), voutput_zero_point);
+
+ int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ const int16x8_t vacc89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF)), voutput_zero_point);
+
+ int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
+#endif
+
+ vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
+
+ vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
+
+ vst1q_s8(output, vout0123456789ABCDEF); output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
+ do {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8((const void*) (k + 136));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8((const void*) (k + 152));
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8((const void*) (k + 168));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8((const void*) (k + 184));
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8((const void*) (k + 200));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8((const void*) (k + 216));
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8((const void*) (k + 232));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8((const void*) (k + 248));
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8((const void*) (k + 264));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8((const void*) (k + 280));
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8((const void*) (k + 296));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8((const void*) (k + 312));
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8((const void*) (k + 328));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8((const void*) (k + 344));
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8((const void*) (k + 360));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8((const void*) (k + 376));
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) ((uintptr_t) w + 8 * sizeof(int32_t) + 400 * sizeof(int8_t)));
+ const float32x4_t vscale4567 = vld1q_f32((const float*) ((uintptr_t) w + 8 * sizeof(int32_t) + 400 * sizeof(int8_t) + 4 * sizeof(float)));
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vacc0123 = vcvtnq_s32_f32(vfpacc0123);
+ vacc4567 = vcvtnq_s32_f32(vfpacc4567);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
+ vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
+
+ if XNN_LIKELY(c >= 8) {
+ vst1_s8(output, vout01234567); output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul8.c b/src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul8.c
index 3433213..225db07 100644
--- a/src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul8.c
+++ b/src/qc8-dwconv/gen/up16x25-minmax-fp32-neonv8-mul8.c
@@ -175,325 +175,325 @@
const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- const int16x8_t vprod0x89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod0x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod0x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- const int16x8_t vprod1x89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod1x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod1x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- const int16x8_t vprod2x89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod2x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod2x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- const int16x8_t vprod3x89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod3x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod3x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- const int16x8_t vprod4x89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod4x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod4x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- const int16x8_t vprod5x89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod5x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod5x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- const int16x8_t vprod6x89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod6x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod6x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- const int16x8_t vprod7x89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod7x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod7x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- const int16x8_t vprod8x89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod8x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod8x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi9x89ABCDEF = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- const int16x8_t vprod9x89ABCDEF = vmull_s8(vi9x89ABCDEF, vk9x89ABCDEF);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod89ABCDEF = vmull_s8(vi9x89ABCDEF, vk9x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod9x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod9x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi10x89ABCDEF = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- const int16x8_t vprod10x89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod10x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod10x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi11x89ABCDEF = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- const int16x8_t vprod11x89ABCDEF = vmull_s8(vi11x89ABCDEF, vk11x89ABCDEF);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod89ABCDEF = vmull_s8(vi11x89ABCDEF, vk11x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod11x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod11x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi12x89ABCDEF = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- const int16x8_t vprod12x89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod12x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod12x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi13x89ABCDEF = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- const int16x8_t vprod13x89ABCDEF = vmull_s8(vi13x89ABCDEF, vk13x89ABCDEF);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod89ABCDEF = vmull_s8(vi13x89ABCDEF, vk13x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod13x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod13x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi14x89ABCDEF = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- const int16x8_t vprod14x89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod14x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod14x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi15x89ABCDEF = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- const int16x8_t vprod15x89ABCDEF = vmull_s8(vi15x89ABCDEF, vk15x89ABCDEF);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod89ABCDEF = vmull_s8(vi15x89ABCDEF, vk15x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod15x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod15x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi16x89ABCDEF = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- const int16x8_t vprod16x89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod16x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod16x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi17x89ABCDEF = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- const int16x8_t vprod17x89ABCDEF = vmull_s8(vi17x89ABCDEF, vk17x89ABCDEF);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod89ABCDEF = vmull_s8(vi17x89ABCDEF, vk17x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod17x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod17x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi18x89ABCDEF = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- const int16x8_t vprod18x89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod18x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod18x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi19x89ABCDEF = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- const int16x8_t vprod19x89ABCDEF = vmull_s8(vi19x89ABCDEF, vk19x89ABCDEF);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod89ABCDEF = vmull_s8(vi19x89ABCDEF, vk19x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod19x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod19x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi20x89ABCDEF = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- const int16x8_t vprod20x89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod20x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod20x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi21x89ABCDEF = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- const int16x8_t vprod21x89ABCDEF = vmull_s8(vi21x89ABCDEF, vk21x89ABCDEF);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod89ABCDEF = vmull_s8(vi21x89ABCDEF, vk21x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod21x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod21x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi22x89ABCDEF = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- const int16x8_t vprod22x89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod22x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod22x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi23x89ABCDEF = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- const int16x8_t vprod23x89ABCDEF = vmull_s8(vi23x89ABCDEF, vk23x89ABCDEF);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod89ABCDEF = vmull_s8(vi23x89ABCDEF, vk23x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod23x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod23x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi24x89ABCDEF = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- const int16x8_t vprod24x89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod24x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod24x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
@@ -542,178 +542,178 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8((const void*) (k + 136));
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8((const void*) (k + 152));
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8((const void*) (k + 168));
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8((const void*) (k + 184));
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8((const void*) (k + 200));
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8((const void*) (k + 216));
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8((const void*) (k + 232));
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8((const void*) (k + 248));
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8((const void*) (k + 264));
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8((const void*) (k + 280));
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8((const void*) (k + 296));
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8((const void*) (k + 312));
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8((const void*) (k + 328));
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8((const void*) (k + 344));
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8((const void*) (k + 360));
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8((const void*) (k + 376));
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mla8.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mla8.c
new file mode 100644
index 0000000..9bf8841
--- /dev/null
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mla8.c
@@ -0,0 +1,357 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const float32x4_t voutput_min_less_zero_point = vld1q_dup_f32(¶ms->neon_fp32.output_min_less_zero_point);
+ const float32x4_t voutput_max_less_zero_point = vld1q_dup_f32(¶ms->neon_fp32.output_max_less_zero_point);
+ const float32x4_t vmagic_bias = vld1q_dup_f32(¶ms->neon_fp32.magic_bias);
+ const int32x4_t vmagic_bias_less_zero_point = vld1q_dup_s32(¶ms->neon_fp32.magic_bias_less_zero_point);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+ float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
+ float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale4567 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale89AB = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscaleCDEF = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+ vfpacc89AB = vmulq_f32(vfpacc89AB, vscale89AB);
+ vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscaleCDEF);
+
+ vfpacc0123 = vmaxq_f32(vfpacc0123, voutput_min_less_zero_point);
+ vfpacc4567 = vmaxq_f32(vfpacc4567, voutput_min_less_zero_point);
+ vfpacc89AB = vmaxq_f32(vfpacc89AB, voutput_min_less_zero_point);
+ vfpaccCDEF = vmaxq_f32(vfpaccCDEF, voutput_min_less_zero_point);
+
+ vfpacc0123 = vminq_f32(vfpacc0123, voutput_max_less_zero_point);
+ vfpacc4567 = vminq_f32(vfpacc4567, voutput_max_less_zero_point);
+ vfpacc89AB = vminq_f32(vfpacc89AB, voutput_max_less_zero_point);
+ vfpaccCDEF = vminq_f32(vfpaccCDEF, voutput_max_less_zero_point);
+
+ vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
+ vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
+ vacc89AB = vreinterpretq_s32_f32(vaddq_f32(vfpacc89AB, vmagic_bias));
+ vaccCDEF = vreinterpretq_s32_f32(vaddq_f32(vfpaccCDEF, vmagic_bias));
+
+ vacc0123 = vsubq_s32(vacc0123, vmagic_bias_less_zero_point);
+ vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
+ vacc89AB = vsubq_s32(vacc89AB, vmagic_bias_less_zero_point);
+ vaccCDEF = vsubq_s32(vaccCDEF, vmagic_bias_less_zero_point);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vuzp1q_s16(vreinterpretq_s16_s32(vacc89AB), vreinterpretq_s16_s32(vaccCDEF));
+
+ int8x16_t vout0123456789ABCDEF = vuzp1q_s8(vreinterpretq_s8_s16(vacc01234567), vreinterpretq_s8_s16(vacc89ABCDEF));
+#else
+ const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
+ const int16x8_t vacc89ABCDEF = vcombine_s16(vmovn_s32(vacc89AB), vmovn_s32(vaccCDEF));
+
+ int8x16_t vout0123456789ABCDEF = vcombine_s8(vmovn_s16(vacc01234567), vmovn_s16(vacc89ABCDEF));
+#endif
+
+
+ vst1q_s8(output, vout0123456789ABCDEF); output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
+ do {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
+ const float32x4_t vscale4567 = vld1q_f32((const float*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t) + 4 * sizeof(float)));
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vfpacc0123 = vmaxq_f32(vfpacc0123, voutput_min_less_zero_point);
+ vfpacc4567 = vmaxq_f32(vfpacc4567, voutput_min_less_zero_point);
+
+ vfpacc0123 = vminq_f32(vfpacc0123, voutput_max_less_zero_point);
+ vfpacc4567 = vminq_f32(vfpacc4567, voutput_max_less_zero_point);
+
+ vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
+ vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
+
+ vacc0123 = vsubq_s32(vacc0123, vmagic_bias_less_zero_point);
+ vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#endif
+
+
+ if XNN_LIKELY(c >= 8) {
+ vst1_s8(output, vout01234567); output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul8.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul8.c
index c495e25..e447084 100644
--- a/src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul8.c
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-neon-mul8.c
@@ -95,117 +95,117 @@
const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- const int16x8_t vprod0x89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod0x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod0x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- const int16x8_t vprod1x89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod1x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod1x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- const int16x8_t vprod2x89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod2x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod2x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- const int16x8_t vprod3x89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod3x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod3x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- const int16x8_t vprod4x89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod4x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod4x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- const int16x8_t vprod5x89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod5x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod5x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- const int16x8_t vprod6x89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod6x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod6x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- const int16x8_t vprod7x89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod7x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod7x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- const int16x8_t vprod8x89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod8x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod8x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
@@ -266,66 +266,66 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mla8.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mla8.c
new file mode 100644
index 0000000..f319fb0
--- /dev/null
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mla8.c
@@ -0,0 +1,338 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/intrinsics-polyfill.h>
+
+
+void xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+ float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
+ float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale4567 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale89AB = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscaleCDEF = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+ vfpacc89AB = vmulq_f32(vfpacc89AB, vscale89AB);
+ vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscaleCDEF);
+
+ vacc0123 = vcvtnq_s32_f32(vfpacc0123);
+ vacc4567 = vcvtnq_s32_f32(vfpacc4567);
+ vacc89AB = vcvtnq_s32_f32(vfpacc89AB);
+ vaccCDEF = vcvtnq_s32_f32(vfpaccCDEF);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ const int16x8_t vacc89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF), voutput_zero_point);
+
+ int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ const int16x8_t vacc89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF)), voutput_zero_point);
+
+ int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
+#endif
+
+ vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
+
+ vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
+
+ vst1q_s8(output, vout0123456789ABCDEF); output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
+ do {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t)));
+ const float32x4_t vscale4567 = vld1q_f32((const float*) ((uintptr_t) w + 8 * sizeof(int32_t) + 144 * sizeof(int8_t) + 4 * sizeof(float)));
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vacc0123 = vcvtnq_s32_f32(vfpacc0123);
+ vacc4567 = vcvtnq_s32_f32(vfpacc4567);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
+ vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
+
+ if XNN_LIKELY(c >= 8) {
+ vst1_s8(output, vout01234567); output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul8.c b/src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul8.c
index 2ff18ee..867d7e5 100644
--- a/src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul8.c
+++ b/src/qc8-dwconv/gen/up16x9-minmax-fp32-neonv8-mul8.c
@@ -95,117 +95,117 @@
const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- const int16x8_t vprod0x89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod0x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod0x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- const int16x8_t vprod1x89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod1x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod1x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- const int16x8_t vprod2x89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod2x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod2x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- const int16x8_t vprod3x89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod3x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod3x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- const int16x8_t vprod4x89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod4x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod4x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- const int16x8_t vprod5x89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod5x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod5x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- const int16x8_t vprod6x89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod6x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod6x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- const int16x8_t vprod7x89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod7x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod7x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- const int16x8_t vprod8x89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod8x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod8x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
@@ -254,66 +254,66 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mla8.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mla8.c
new file mode 100644
index 0000000..f69791c
--- /dev/null
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mla8.c
@@ -0,0 +1,583 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const float32x4_t voutput_min_less_zero_point = vld1q_dup_f32(¶ms->neon_fp32.output_min_less_zero_point);
+ const float32x4_t voutput_max_less_zero_point = vld1q_dup_f32(¶ms->neon_fp32.output_max_less_zero_point);
+ const float32x4_t vmagic_bias = vld1q_dup_f32(¶ms->neon_fp32.magic_bias);
+ const int32x4_t vmagic_bias_less_zero_point = vld1q_dup_s32(¶ms->neon_fp32.magic_bias_less_zero_point);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 8; c -= 8) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale4567 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vfpacc0123 = vmaxq_f32(vfpacc0123, voutput_min_less_zero_point);
+ vfpacc4567 = vmaxq_f32(vfpacc4567, voutput_min_less_zero_point);
+
+ vfpacc0123 = vminq_f32(vfpacc0123, voutput_max_less_zero_point);
+ vfpacc4567 = vminq_f32(vfpacc4567, voutput_max_less_zero_point);
+
+ vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
+ vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
+
+ vacc0123 = vsubq_s32(vacc0123, vmagic_bias_less_zero_point);
+ vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
+
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#endif
+
+
+ vst1_s8(output, vout01234567); output += 8;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0);
+ const int8x8_t vk0x01234567 = vld1_s8(w);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1);
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2);
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3);
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4);
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5);
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6);
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7);
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8);
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9);
+ const int8x8_t vk9x01234567 = vld1_s8((const void*) ((const int8_t*) w + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi10x01234567 = vld1_s8(i10);
+ const int8x8_t vk10x01234567 = vld1_s8((const void*) ((const int8_t*) w + 80));
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11);
+ const int8x8_t vk11x01234567 = vld1_s8((const void*) ((const int8_t*) w + 88));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi12x01234567 = vld1_s8(i12);
+ const int8x8_t vk12x01234567 = vld1_s8((const void*) ((const int8_t*) w + 96));
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13);
+ const int8x8_t vk13x01234567 = vld1_s8((const void*) ((const int8_t*) w + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi14x01234567 = vld1_s8(i14);
+ const int8x8_t vk14x01234567 = vld1_s8((const void*) ((const int8_t*) w + 112));
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15);
+ const int8x8_t vk15x01234567 = vld1_s8((const void*) ((const int8_t*) w + 120));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi16x01234567 = vld1_s8(i16);
+ const int8x8_t vk16x01234567 = vld1_s8((const void*) ((const int8_t*) w + 128));
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17);
+ const int8x8_t vk17x01234567 = vld1_s8((const void*) ((const int8_t*) w + 136));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi18x01234567 = vld1_s8(i18);
+ const int8x8_t vk18x01234567 = vld1_s8((const void*) ((const int8_t*) w + 144));
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19);
+ const int8x8_t vk19x01234567 = vld1_s8((const void*) ((const int8_t*) w + 152));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi20x01234567 = vld1_s8(i20);
+ const int8x8_t vk20x01234567 = vld1_s8((const void*) ((const int8_t*) w + 160));
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21);
+ const int8x8_t vk21x01234567 = vld1_s8((const void*) ((const int8_t*) w + 168));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi22x01234567 = vld1_s8(i22);
+ const int8x8_t vk22x01234567 = vld1_s8((const void*) ((const int8_t*) w + 176));
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23);
+ const int8x8_t vk23x01234567 = vld1_s8((const void*) ((const int8_t*) w + 184));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi24x01234567 = vld1_s8(i24);
+ const int8x8_t vk24x01234567 = vld1_s8((const void*) ((const int8_t*) w + 192));
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) ((uintptr_t) w + 0 * sizeof(int32_t) + 200 * sizeof(int8_t)));
+ const float32x4_t vscale4567 = vld1q_f32((const float*) ((uintptr_t) w + 0 * sizeof(int32_t) + 200 * sizeof(int8_t) + 4 * sizeof(float)));
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vfpacc0123 = vmaxq_f32(vfpacc0123, voutput_min_less_zero_point);
+ vfpacc4567 = vmaxq_f32(vfpacc4567, voutput_min_less_zero_point);
+
+ vfpacc0123 = vminq_f32(vfpacc0123, voutput_max_less_zero_point);
+ vfpacc4567 = vminq_f32(vfpacc4567, voutput_max_less_zero_point);
+
+ vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
+ vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
+
+ vacc0123 = vsubq_s32(vacc0123, vmagic_bias_less_zero_point);
+ vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#endif
+
+
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul8.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul8.c
index 6267503..05f2d39 100644
--- a/src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul8.c
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-neon-mul8.c
@@ -171,202 +171,202 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
@@ -410,178 +410,178 @@
const int8x8_t vi0x01234567 = vld1_s8(i0);
const int8x8_t vk0x01234567 = vld1_s8(w);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1);
const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2);
const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3);
const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4);
const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5);
const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6);
const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7);
const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8);
const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9);
const int8x8_t vk9x01234567 = vld1_s8((const void*) ((const int8_t*) w + 72));
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10);
const int8x8_t vk10x01234567 = vld1_s8((const void*) ((const int8_t*) w + 80));
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11);
const int8x8_t vk11x01234567 = vld1_s8((const void*) ((const int8_t*) w + 88));
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12);
const int8x8_t vk12x01234567 = vld1_s8((const void*) ((const int8_t*) w + 96));
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13);
const int8x8_t vk13x01234567 = vld1_s8((const void*) ((const int8_t*) w + 104));
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14);
const int8x8_t vk14x01234567 = vld1_s8((const void*) ((const int8_t*) w + 112));
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15);
const int8x8_t vk15x01234567 = vld1_s8((const void*) ((const int8_t*) w + 120));
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16);
const int8x8_t vk16x01234567 = vld1_s8((const void*) ((const int8_t*) w + 128));
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17);
const int8x8_t vk17x01234567 = vld1_s8((const void*) ((const int8_t*) w + 136));
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18);
const int8x8_t vk18x01234567 = vld1_s8((const void*) ((const int8_t*) w + 144));
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19);
const int8x8_t vk19x01234567 = vld1_s8((const void*) ((const int8_t*) w + 152));
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20);
const int8x8_t vk20x01234567 = vld1_s8((const void*) ((const int8_t*) w + 160));
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21);
const int8x8_t vk21x01234567 = vld1_s8((const void*) ((const int8_t*) w + 168));
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22);
const int8x8_t vk22x01234567 = vld1_s8((const void*) ((const int8_t*) w + 176));
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23);
const int8x8_t vk23x01234567 = vld1_s8((const void*) ((const int8_t*) w + 184));
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24);
const int8x8_t vk24x01234567 = vld1_s8((const void*) ((const int8_t*) w + 192));
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mla8.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mla8.c
new file mode 100644
index 0000000..1109ba9
--- /dev/null
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mla8.c
@@ -0,0 +1,570 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/intrinsics-polyfill.h>
+
+
+void xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 8; c -= 8) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale4567 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vacc0123 = vcvtnq_s32_f32(vfpacc0123);
+ vacc4567 = vcvtnq_s32_f32(vfpacc4567);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, voutput_min);
+
+ vout01234567 = vmin_s8(vout01234567, voutput_max);
+
+ vst1_s8(output, vout01234567); output += 8;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0);
+ const int8x8_t vk0x01234567 = vld1_s8(w);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1);
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2);
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3);
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4);
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5);
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6);
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7);
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8);
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9);
+ const int8x8_t vk9x01234567 = vld1_s8((const void*) ((const int8_t*) w + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi10x01234567 = vld1_s8(i10);
+ const int8x8_t vk10x01234567 = vld1_s8((const void*) ((const int8_t*) w + 80));
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11);
+ const int8x8_t vk11x01234567 = vld1_s8((const void*) ((const int8_t*) w + 88));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi12x01234567 = vld1_s8(i12);
+ const int8x8_t vk12x01234567 = vld1_s8((const void*) ((const int8_t*) w + 96));
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13);
+ const int8x8_t vk13x01234567 = vld1_s8((const void*) ((const int8_t*) w + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi14x01234567 = vld1_s8(i14);
+ const int8x8_t vk14x01234567 = vld1_s8((const void*) ((const int8_t*) w + 112));
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15);
+ const int8x8_t vk15x01234567 = vld1_s8((const void*) ((const int8_t*) w + 120));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi16x01234567 = vld1_s8(i16);
+ const int8x8_t vk16x01234567 = vld1_s8((const void*) ((const int8_t*) w + 128));
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17);
+ const int8x8_t vk17x01234567 = vld1_s8((const void*) ((const int8_t*) w + 136));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi18x01234567 = vld1_s8(i18);
+ const int8x8_t vk18x01234567 = vld1_s8((const void*) ((const int8_t*) w + 144));
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19);
+ const int8x8_t vk19x01234567 = vld1_s8((const void*) ((const int8_t*) w + 152));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi20x01234567 = vld1_s8(i20);
+ const int8x8_t vk20x01234567 = vld1_s8((const void*) ((const int8_t*) w + 160));
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21);
+ const int8x8_t vk21x01234567 = vld1_s8((const void*) ((const int8_t*) w + 168));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi22x01234567 = vld1_s8(i22);
+ const int8x8_t vk22x01234567 = vld1_s8((const void*) ((const int8_t*) w + 176));
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23);
+ const int8x8_t vk23x01234567 = vld1_s8((const void*) ((const int8_t*) w + 184));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi24x01234567 = vld1_s8(i24);
+ const int8x8_t vk24x01234567 = vld1_s8((const void*) ((const int8_t*) w + 192));
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) ((uintptr_t) w + 0 * sizeof(int32_t) + 200 * sizeof(int8_t)));
+ const float32x4_t vscale4567 = vld1q_f32((const float*) ((uintptr_t) w + 0 * sizeof(int32_t) + 200 * sizeof(int8_t) + 4 * sizeof(float)));
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vacc0123 = vcvtnq_s32_f32(vfpacc0123);
+ vacc4567 = vcvtnq_s32_f32(vfpacc4567);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, voutput_min);
+ vout01234567 = vmin_s8(vout01234567, voutput_max);
+
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul8.c b/src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul8.c
index a9232a8..5a8df23 100644
--- a/src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul8.c
+++ b/src/qc8-dwconv/gen/up8x25-minmax-fp32-neonv8-mul8.c
@@ -171,202 +171,202 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
@@ -404,178 +404,178 @@
const int8x8_t vi0x01234567 = vld1_s8(i0);
const int8x8_t vk0x01234567 = vld1_s8(w);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1);
const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2);
const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3);
const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4);
const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5);
const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6);
const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7);
const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8);
const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9);
const int8x8_t vk9x01234567 = vld1_s8((const void*) ((const int8_t*) w + 72));
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10);
const int8x8_t vk10x01234567 = vld1_s8((const void*) ((const int8_t*) w + 80));
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11);
const int8x8_t vk11x01234567 = vld1_s8((const void*) ((const int8_t*) w + 88));
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12);
const int8x8_t vk12x01234567 = vld1_s8((const void*) ((const int8_t*) w + 96));
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13);
const int8x8_t vk13x01234567 = vld1_s8((const void*) ((const int8_t*) w + 104));
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14);
const int8x8_t vk14x01234567 = vld1_s8((const void*) ((const int8_t*) w + 112));
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15);
const int8x8_t vk15x01234567 = vld1_s8((const void*) ((const int8_t*) w + 120));
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16);
const int8x8_t vk16x01234567 = vld1_s8((const void*) ((const int8_t*) w + 128));
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17);
const int8x8_t vk17x01234567 = vld1_s8((const void*) ((const int8_t*) w + 136));
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18);
const int8x8_t vk18x01234567 = vld1_s8((const void*) ((const int8_t*) w + 144));
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19);
const int8x8_t vk19x01234567 = vld1_s8((const void*) ((const int8_t*) w + 152));
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20);
const int8x8_t vk20x01234567 = vld1_s8((const void*) ((const int8_t*) w + 160));
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21);
const int8x8_t vk21x01234567 = vld1_s8((const void*) ((const int8_t*) w + 168));
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22);
const int8x8_t vk22x01234567 = vld1_s8((const void*) ((const int8_t*) w + 176));
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23);
const int8x8_t vk23x01234567 = vld1_s8((const void*) ((const int8_t*) w + 184));
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24);
const int8x8_t vk24x01234567 = vld1_s8((const void*) ((const int8_t*) w + 192));
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mla8.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mla8.c
new file mode 100644
index 0000000..9229eed
--- /dev/null
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mla8.c
@@ -0,0 +1,295 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const float32x4_t voutput_min_less_zero_point = vld1q_dup_f32(¶ms->neon_fp32.output_min_less_zero_point);
+ const float32x4_t voutput_max_less_zero_point = vld1q_dup_f32(¶ms->neon_fp32.output_max_less_zero_point);
+ const float32x4_t vmagic_bias = vld1q_dup_f32(¶ms->neon_fp32.magic_bias);
+ const int32x4_t vmagic_bias_less_zero_point = vld1q_dup_s32(¶ms->neon_fp32.magic_bias_less_zero_point);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 8; c -= 8) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale4567 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vfpacc0123 = vmaxq_f32(vfpacc0123, voutput_min_less_zero_point);
+ vfpacc4567 = vmaxq_f32(vfpacc4567, voutput_min_less_zero_point);
+
+ vfpacc0123 = vminq_f32(vfpacc0123, voutput_max_less_zero_point);
+ vfpacc4567 = vminq_f32(vfpacc4567, voutput_max_less_zero_point);
+
+ vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
+ vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
+
+ vacc0123 = vsubq_s32(vacc0123, vmagic_bias_less_zero_point);
+ vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
+
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#endif
+
+
+ vst1_s8(output, vout01234567); output += 8;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0);
+ const int8x8_t vk0x01234567 = vld1_s8(w);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1);
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2);
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3);
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4);
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5);
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6);
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7);
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8);
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) ((uintptr_t) w + 0 * sizeof(int32_t) + 72 * sizeof(int8_t)));
+ const float32x4_t vscale4567 = vld1q_f32((const float*) ((uintptr_t) w + 0 * sizeof(int32_t) + 72 * sizeof(int8_t) + 4 * sizeof(float)));
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vfpacc0123 = vmaxq_f32(vfpacc0123, voutput_min_less_zero_point);
+ vfpacc4567 = vmaxq_f32(vfpacc4567, voutput_min_less_zero_point);
+
+ vfpacc0123 = vminq_f32(vfpacc0123, voutput_max_less_zero_point);
+ vfpacc4567 = vminq_f32(vfpacc4567, voutput_max_less_zero_point);
+
+ vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
+ vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
+
+ vacc0123 = vsubq_s32(vacc0123, vmagic_bias_less_zero_point);
+ vacc4567 = vsubq_s32(vacc4567, vmagic_bias_less_zero_point);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vuzp1q_s16(vreinterpretq_s16_s32(vacc0123), vreinterpretq_s16_s32(vacc4567));
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vcombine_s16(vmovn_s32(vacc0123), vmovn_s32(vacc4567));
+ int8x8_t vout01234567 = vmovn_s16(vacc01234567);
+#endif
+
+
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul8.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul8.c
index 57e33b9..ff109a4 100644
--- a/src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul8.c
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-neon-mul8.c
@@ -91,74 +91,74 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
@@ -202,66 +202,66 @@
const int8x8_t vi0x01234567 = vld1_s8(i0);
const int8x8_t vk0x01234567 = vld1_s8(w);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1);
const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2);
const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3);
const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4);
const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5);
const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6);
const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7);
const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8);
const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mla8.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mla8.c
new file mode 100644
index 0000000..49b5c0a
--- /dev/null
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mla8.c
@@ -0,0 +1,282 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+#include <xnnpack/intrinsics-polyfill.h>
+
+
+void xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->neon.output_max);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 8; c -= 8) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+ const float32x4_t vscale4567 = vld1q_f32((const float*) w); w = (const void*) ((const float*) w + 4);
+
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vacc0123 = vcvtnq_s32_f32(vfpacc0123);
+ vacc4567 = vcvtnq_s32_f32(vfpacc4567);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, voutput_min);
+
+ vout01234567 = vmin_s8(vout01234567, voutput_max);
+
+ vst1_s8(output, vout01234567); output += 8;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0);
+ const int8x8_t vk0x01234567 = vld1_s8(w);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1);
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2);
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3);
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4);
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5);
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6);
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7);
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8);
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
+ float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
+
+ const float32x4_t vscale0123 = vld1q_f32((const float*) ((uintptr_t) w + 0 * sizeof(int32_t) + 72 * sizeof(int8_t)));
+ const float32x4_t vscale4567 = vld1q_f32((const float*) ((uintptr_t) w + 0 * sizeof(int32_t) + 72 * sizeof(int8_t) + 4 * sizeof(float)));
+ vfpacc0123 = vmulq_f32(vfpacc0123, vscale0123);
+ vfpacc4567 = vmulq_f32(vfpacc4567, vscale4567);
+
+ vacc0123 = vcvtnq_s32_f32(vfpacc0123);
+ vacc4567 = vcvtnq_s32_f32(vfpacc4567);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, voutput_min);
+ vout01234567 = vmin_s8(vout01234567, voutput_max);
+
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul8.c b/src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul8.c
index 231e47e..ee2418c 100644
--- a/src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul8.c
+++ b/src/qc8-dwconv/gen/up8x9-minmax-fp32-neonv8-mul8.c
@@ -91,74 +91,74 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
@@ -196,66 +196,66 @@
const int8x8_t vi0x01234567 = vld1_s8(i0);
const int8x8_t vk0x01234567 = vld1_s8(w);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1);
const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2);
const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3);
const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4);
const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5);
const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6);
const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7);
const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8);
const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mla8.c b/src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mla8.c
new file mode 100644
index 0000000..33be6a3
--- /dev/null
+++ b/src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mla8.c
@@ -0,0 +1,685 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_pre_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->rndnu_neon.multiplier);
+ const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_post_shift);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->rndnu_neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->rndnu_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->rndnu_neon.output_max);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi9x89ABCDEF = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi9x89ABCDEF, vk9x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi10x89ABCDEF = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
+
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi11x89ABCDEF = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi11x89ABCDEF, vk11x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi12x89ABCDEF = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
+
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi13x89ABCDEF = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi13x89ABCDEF, vk13x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi14x89ABCDEF = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
+
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi15x89ABCDEF = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi15x89ABCDEF, vk15x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi16x89ABCDEF = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
+
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi17x89ABCDEF = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi17x89ABCDEF, vk17x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi18x89ABCDEF = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
+
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi19x89ABCDEF = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi19x89ABCDEF, vk19x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi20x89ABCDEF = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
+
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi21x89ABCDEF = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi21x89ABCDEF, vk21x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi22x89ABCDEF = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
+
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi23x89ABCDEF = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi23x89ABCDEF, vk23x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi24x89ABCDEF = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
+ vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
+ vacc89AB = vshlq_s32(vacc89AB, vright_pre_shift);
+ vaccCDEF = vshlq_s32(vaccCDEF, vright_pre_shift);
+
+ vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
+ vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
+ vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
+ vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
+ vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
+ vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ const int16x8_t vacc89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF), voutput_zero_point);
+
+ int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ const int16x8_t vacc89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF)), voutput_zero_point);
+
+ int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
+#endif
+
+ vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
+
+ vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
+
+ vst1q_s8(output, vout0123456789ABCDEF); output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
+ do {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8((const void*) (k + 136));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8((const void*) (k + 152));
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8((const void*) (k + 168));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8((const void*) (k + 184));
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8((const void*) (k + 200));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8((const void*) (k + 216));
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8((const void*) (k + 232));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8((const void*) (k + 248));
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8((const void*) (k + 264));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8((const void*) (k + 280));
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8((const void*) (k + 296));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8((const void*) (k + 312));
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8((const void*) (k + 328));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8((const void*) (k + 344));
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8((const void*) (k + 360));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8((const void*) (k + 376));
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
+
+ vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
+ vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
+ vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
+
+ if XNN_LIKELY(c >= 8) {
+ vst1_s8(output, vout01234567); output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul8.c b/src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul8.c
index 2398864..7aca8d3 100644
--- a/src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul8.c
+++ b/src/qs8-dwconv/gen/up16x25-minmax-rndnu-neon-mul8.c
@@ -177,325 +177,325 @@
const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- const int16x8_t vprod0x89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod0x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod0x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- const int16x8_t vprod1x89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod1x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod1x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- const int16x8_t vprod2x89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod2x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod2x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- const int16x8_t vprod3x89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod3x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod3x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- const int16x8_t vprod4x89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod4x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod4x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- const int16x8_t vprod5x89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod5x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod5x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- const int16x8_t vprod6x89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod6x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod6x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- const int16x8_t vprod7x89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod7x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod7x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- const int16x8_t vprod8x89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod8x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod8x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi9x89ABCDEF = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- const int16x8_t vprod9x89ABCDEF = vmull_s8(vi9x89ABCDEF, vk9x89ABCDEF);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod89ABCDEF = vmull_s8(vi9x89ABCDEF, vk9x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod9x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod9x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi10x89ABCDEF = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- const int16x8_t vprod10x89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod10x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod10x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi11x89ABCDEF = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- const int16x8_t vprod11x89ABCDEF = vmull_s8(vi11x89ABCDEF, vk11x89ABCDEF);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod89ABCDEF = vmull_s8(vi11x89ABCDEF, vk11x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod11x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod11x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi12x89ABCDEF = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- const int16x8_t vprod12x89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod12x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod12x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi13x89ABCDEF = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- const int16x8_t vprod13x89ABCDEF = vmull_s8(vi13x89ABCDEF, vk13x89ABCDEF);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod89ABCDEF = vmull_s8(vi13x89ABCDEF, vk13x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod13x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod13x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi14x89ABCDEF = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- const int16x8_t vprod14x89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod14x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod14x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi15x89ABCDEF = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- const int16x8_t vprod15x89ABCDEF = vmull_s8(vi15x89ABCDEF, vk15x89ABCDEF);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod89ABCDEF = vmull_s8(vi15x89ABCDEF, vk15x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod15x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod15x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi16x89ABCDEF = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- const int16x8_t vprod16x89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod16x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod16x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi17x89ABCDEF = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- const int16x8_t vprod17x89ABCDEF = vmull_s8(vi17x89ABCDEF, vk17x89ABCDEF);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod89ABCDEF = vmull_s8(vi17x89ABCDEF, vk17x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod17x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod17x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi18x89ABCDEF = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- const int16x8_t vprod18x89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod18x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod18x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi19x89ABCDEF = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- const int16x8_t vprod19x89ABCDEF = vmull_s8(vi19x89ABCDEF, vk19x89ABCDEF);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod89ABCDEF = vmull_s8(vi19x89ABCDEF, vk19x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod19x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod19x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi20x89ABCDEF = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- const int16x8_t vprod20x89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod20x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod20x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi21x89ABCDEF = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- const int16x8_t vprod21x89ABCDEF = vmull_s8(vi21x89ABCDEF, vk21x89ABCDEF);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod89ABCDEF = vmull_s8(vi21x89ABCDEF, vk21x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod21x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod21x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi22x89ABCDEF = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- const int16x8_t vprod22x89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod22x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod22x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi23x89ABCDEF = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- const int16x8_t vprod23x89ABCDEF = vmull_s8(vi23x89ABCDEF, vk23x89ABCDEF);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod89ABCDEF = vmull_s8(vi23x89ABCDEF, vk23x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod23x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod23x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi24x89ABCDEF = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- const int16x8_t vprod24x89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod24x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod24x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
@@ -539,178 +539,178 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8((const void*) (k + 136));
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8((const void*) (k + 152));
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8((const void*) (k + 168));
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8((const void*) (k + 184));
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8((const void*) (k + 200));
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8((const void*) (k + 216));
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8((const void*) (k + 232));
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8((const void*) (k + 248));
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8((const void*) (k + 264));
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8((const void*) (k + 280));
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8((const void*) (k + 296));
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8((const void*) (k + 312));
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8((const void*) (k + 328));
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8((const void*) (k + 344));
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8((const void*) (k + 360));
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8((const void*) (k + 376));
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mla8.c b/src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mla8.c
new file mode 100644
index 0000000..a155bc4
--- /dev/null
+++ b/src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mla8.c
@@ -0,0 +1,333 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_pre_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->rndnu_neon.multiplier);
+ const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_post_shift);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->rndnu_neon.output_zero_point);
+ const int8x16_t voutput_min = vld1q_dup_s8(¶ms->rndnu_neon.output_min);
+ const int8x16_t voutput_max = vld1q_dup_s8(¶ms->rndnu_neon.output_max);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 16; c -= 16) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+ const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
+
+ vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
+ vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
+ vacc89AB = vshlq_s32(vacc89AB, vright_pre_shift);
+ vaccCDEF = vshlq_s32(vaccCDEF, vright_pre_shift);
+
+ vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
+ vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
+ vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
+ vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
+ vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
+ vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ const int16x8_t vacc89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF), voutput_zero_point);
+
+ int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ const int16x8_t vacc89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF)), voutput_zero_point);
+
+ int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
+#endif
+
+ vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
+
+ vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
+
+ vst1q_s8(output, vout0123456789ABCDEF); output += 16;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
+ do {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
+
+ vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
+ vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
+ vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
+
+ if XNN_LIKELY(c >= 8) {
+ vst1_s8(output, vout01234567); output += 8;
+ c -= 8;
+ } else {
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ c = 0;
+ }
+ } while (c != 0);
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul8.c b/src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul8.c
index 68f1940..cef1806 100644
--- a/src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul8.c
+++ b/src/qs8-dwconv/gen/up16x9-minmax-rndnu-neon-mul8.c
@@ -97,117 +97,117 @@
const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- const int16x8_t vprod0x89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod0x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod0x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- const int16x8_t vprod1x89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod1x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod1x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- const int16x8_t vprod2x89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod2x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod2x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- const int16x8_t vprod3x89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod3x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod3x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- const int16x8_t vprod4x89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod4x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod4x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- const int16x8_t vprod5x89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod5x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod5x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- const int16x8_t vprod6x89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod6x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod6x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- const int16x8_t vprod7x89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod7x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod7x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- const int16x8_t vprod8x89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
- vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod8x89ABCDEF));
- vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod8x89ABCDEF));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
+ vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
@@ -251,66 +251,66 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mla8.c b/src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mla8.c
new file mode 100644
index 0000000..86b035f
--- /dev/null
+++ b/src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mla8.c
@@ -0,0 +1,567 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_pre_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->rndnu_neon.multiplier);
+ const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_post_shift);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->rndnu_neon.output_zero_point);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->rndnu_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->rndnu_neon.output_max);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ const int8_t* i9 = input[9];
+ assert(i9 != NULL);
+ if XNN_UNPREDICTABLE(i9 != zero) {
+ i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
+ }
+ const int8_t* i10 = input[10];
+ assert(i10 != NULL);
+ if XNN_UNPREDICTABLE(i10 != zero) {
+ i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
+ }
+ const int8_t* i11 = input[11];
+ assert(i11 != NULL);
+ if XNN_UNPREDICTABLE(i11 != zero) {
+ i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
+ }
+ const int8_t* i12 = input[12];
+ assert(i12 != NULL);
+ if XNN_UNPREDICTABLE(i12 != zero) {
+ i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
+ }
+ const int8_t* i13 = input[13];
+ assert(i13 != NULL);
+ if XNN_UNPREDICTABLE(i13 != zero) {
+ i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
+ }
+ const int8_t* i14 = input[14];
+ assert(i14 != NULL);
+ if XNN_UNPREDICTABLE(i14 != zero) {
+ i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
+ }
+ const int8_t* i15 = input[15];
+ assert(i15 != NULL);
+ if XNN_UNPREDICTABLE(i15 != zero) {
+ i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
+ }
+ const int8_t* i16 = input[16];
+ assert(i16 != NULL);
+ if XNN_UNPREDICTABLE(i16 != zero) {
+ i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
+ }
+ const int8_t* i17 = input[17];
+ assert(i17 != NULL);
+ if XNN_UNPREDICTABLE(i17 != zero) {
+ i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
+ }
+ const int8_t* i18 = input[18];
+ assert(i18 != NULL);
+ if XNN_UNPREDICTABLE(i18 != zero) {
+ i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
+ }
+ const int8_t* i19 = input[19];
+ assert(i19 != NULL);
+ if XNN_UNPREDICTABLE(i19 != zero) {
+ i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
+ }
+ const int8_t* i20 = input[20];
+ assert(i20 != NULL);
+ if XNN_UNPREDICTABLE(i20 != zero) {
+ i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
+ }
+ const int8_t* i21 = input[21];
+ assert(i21 != NULL);
+ if XNN_UNPREDICTABLE(i21 != zero) {
+ i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
+ }
+ const int8_t* i22 = input[22];
+ assert(i22 != NULL);
+ if XNN_UNPREDICTABLE(i22 != zero) {
+ i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
+ }
+ const int8_t* i23 = input[23];
+ assert(i23 != NULL);
+ if XNN_UNPREDICTABLE(i23 != zero) {
+ i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
+ }
+ const int8_t* i24 = input[24];
+ assert(i24 != NULL);
+ if XNN_UNPREDICTABLE(i24 != zero) {
+ i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 8; c -= 8) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
+ const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
+ const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
+ const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
+ const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
+ const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
+ const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
+ const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
+ const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
+ const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
+ const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
+ const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
+ const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
+ const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
+ const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
+ const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
+ const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
+ vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
+
+ vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
+ vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, voutput_min);
+
+ vout01234567 = vmin_s8(vout01234567, voutput_max);
+
+ vst1_s8(output, vout01234567); output += 8;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0);
+ const int8x8_t vk0x01234567 = vld1_s8(w);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1);
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2);
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3);
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4);
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5);
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6);
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7);
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8);
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ const int8x8_t vi9x01234567 = vld1_s8(i9);
+ const int8x8_t vk9x01234567 = vld1_s8((const void*) ((const int8_t*) w + 72));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi9x01234567, vk9x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi10x01234567 = vld1_s8(i10);
+ const int8x8_t vk10x01234567 = vld1_s8((const void*) ((const int8_t*) w + 80));
+
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+
+ const int8x8_t vi11x01234567 = vld1_s8(i11);
+ const int8x8_t vk11x01234567 = vld1_s8((const void*) ((const int8_t*) w + 88));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi11x01234567, vk11x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi12x01234567 = vld1_s8(i12);
+ const int8x8_t vk12x01234567 = vld1_s8((const void*) ((const int8_t*) w + 96));
+
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+
+ const int8x8_t vi13x01234567 = vld1_s8(i13);
+ const int8x8_t vk13x01234567 = vld1_s8((const void*) ((const int8_t*) w + 104));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi13x01234567, vk13x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi14x01234567 = vld1_s8(i14);
+ const int8x8_t vk14x01234567 = vld1_s8((const void*) ((const int8_t*) w + 112));
+
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+
+ const int8x8_t vi15x01234567 = vld1_s8(i15);
+ const int8x8_t vk15x01234567 = vld1_s8((const void*) ((const int8_t*) w + 120));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi15x01234567, vk15x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi16x01234567 = vld1_s8(i16);
+ const int8x8_t vk16x01234567 = vld1_s8((const void*) ((const int8_t*) w + 128));
+
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+
+ const int8x8_t vi17x01234567 = vld1_s8(i17);
+ const int8x8_t vk17x01234567 = vld1_s8((const void*) ((const int8_t*) w + 136));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi17x01234567, vk17x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi18x01234567 = vld1_s8(i18);
+ const int8x8_t vk18x01234567 = vld1_s8((const void*) ((const int8_t*) w + 144));
+
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+
+ const int8x8_t vi19x01234567 = vld1_s8(i19);
+ const int8x8_t vk19x01234567 = vld1_s8((const void*) ((const int8_t*) w + 152));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi19x01234567, vk19x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi20x01234567 = vld1_s8(i20);
+ const int8x8_t vk20x01234567 = vld1_s8((const void*) ((const int8_t*) w + 160));
+
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+
+ const int8x8_t vi21x01234567 = vld1_s8(i21);
+ const int8x8_t vk21x01234567 = vld1_s8((const void*) ((const int8_t*) w + 168));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi21x01234567, vk21x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi22x01234567 = vld1_s8(i22);
+ const int8x8_t vk22x01234567 = vld1_s8((const void*) ((const int8_t*) w + 176));
+
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+
+ const int8x8_t vi23x01234567 = vld1_s8(i23);
+ const int8x8_t vk23x01234567 = vld1_s8((const void*) ((const int8_t*) w + 184));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi23x01234567, vk23x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi24x01234567 = vld1_s8(i24);
+ const int8x8_t vk24x01234567 = vld1_s8((const void*) ((const int8_t*) w + 192));
+
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
+
+ vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
+ vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, voutput_min);
+ vout01234567 = vmin_s8(vout01234567, voutput_max);
+
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c b/src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c
index 5e7a02b..b4d2a09 100644
--- a/src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c
+++ b/src/qs8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c
@@ -173,202 +173,202 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
@@ -403,178 +403,178 @@
const int8x8_t vi0x01234567 = vld1_s8(i0);
const int8x8_t vk0x01234567 = vld1_s8(w);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1);
const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2);
const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3);
const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4);
const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5);
const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6);
const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7);
const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8);
const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi9x01234567 = vld1_s8(i9);
const int8x8_t vk9x01234567 = vld1_s8((const void*) ((const int8_t*) w + 72));
- const int16x8_t vprod9x01234567 = vmull_s8(vi9x01234567, vk9x01234567);
+ vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod9x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod9x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi10x01234567 = vld1_s8(i10);
const int8x8_t vk10x01234567 = vld1_s8((const void*) ((const int8_t*) w + 80));
- const int16x8_t vprod10x01234567 = vmull_s8(vi10x01234567, vk10x01234567);
+ vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod10x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod10x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi11x01234567 = vld1_s8(i11);
const int8x8_t vk11x01234567 = vld1_s8((const void*) ((const int8_t*) w + 88));
- const int16x8_t vprod11x01234567 = vmull_s8(vi11x01234567, vk11x01234567);
+ vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod11x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod11x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi12x01234567 = vld1_s8(i12);
const int8x8_t vk12x01234567 = vld1_s8((const void*) ((const int8_t*) w + 96));
- const int16x8_t vprod12x01234567 = vmull_s8(vi12x01234567, vk12x01234567);
+ vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod12x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod12x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi13x01234567 = vld1_s8(i13);
const int8x8_t vk13x01234567 = vld1_s8((const void*) ((const int8_t*) w + 104));
- const int16x8_t vprod13x01234567 = vmull_s8(vi13x01234567, vk13x01234567);
+ vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod13x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod13x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi14x01234567 = vld1_s8(i14);
const int8x8_t vk14x01234567 = vld1_s8((const void*) ((const int8_t*) w + 112));
- const int16x8_t vprod14x01234567 = vmull_s8(vi14x01234567, vk14x01234567);
+ vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod14x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod14x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi15x01234567 = vld1_s8(i15);
const int8x8_t vk15x01234567 = vld1_s8((const void*) ((const int8_t*) w + 120));
- const int16x8_t vprod15x01234567 = vmull_s8(vi15x01234567, vk15x01234567);
+ vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod15x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod15x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi16x01234567 = vld1_s8(i16);
const int8x8_t vk16x01234567 = vld1_s8((const void*) ((const int8_t*) w + 128));
- const int16x8_t vprod16x01234567 = vmull_s8(vi16x01234567, vk16x01234567);
+ vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod16x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod16x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi17x01234567 = vld1_s8(i17);
const int8x8_t vk17x01234567 = vld1_s8((const void*) ((const int8_t*) w + 136));
- const int16x8_t vprod17x01234567 = vmull_s8(vi17x01234567, vk17x01234567);
+ vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod17x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod17x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi18x01234567 = vld1_s8(i18);
const int8x8_t vk18x01234567 = vld1_s8((const void*) ((const int8_t*) w + 144));
- const int16x8_t vprod18x01234567 = vmull_s8(vi18x01234567, vk18x01234567);
+ vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod18x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod18x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi19x01234567 = vld1_s8(i19);
const int8x8_t vk19x01234567 = vld1_s8((const void*) ((const int8_t*) w + 152));
- const int16x8_t vprod19x01234567 = vmull_s8(vi19x01234567, vk19x01234567);
+ vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod19x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod19x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi20x01234567 = vld1_s8(i20);
const int8x8_t vk20x01234567 = vld1_s8((const void*) ((const int8_t*) w + 160));
- const int16x8_t vprod20x01234567 = vmull_s8(vi20x01234567, vk20x01234567);
+ vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod20x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod20x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi21x01234567 = vld1_s8(i21);
const int8x8_t vk21x01234567 = vld1_s8((const void*) ((const int8_t*) w + 168));
- const int16x8_t vprod21x01234567 = vmull_s8(vi21x01234567, vk21x01234567);
+ vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod21x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod21x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi22x01234567 = vld1_s8(i22);
const int8x8_t vk22x01234567 = vld1_s8((const void*) ((const int8_t*) w + 176));
- const int16x8_t vprod22x01234567 = vmull_s8(vi22x01234567, vk22x01234567);
+ vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod22x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod22x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi23x01234567 = vld1_s8(i23);
const int8x8_t vk23x01234567 = vld1_s8((const void*) ((const int8_t*) w + 184));
- const int16x8_t vprod23x01234567 = vmull_s8(vi23x01234567, vk23x01234567);
+ vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod23x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod23x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi24x01234567 = vld1_s8(i24);
const int8x8_t vk24x01234567 = vld1_s8((const void*) ((const int8_t*) w + 192));
- const int16x8_t vprod24x01234567 = vmull_s8(vi24x01234567, vk24x01234567);
+ vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod24x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod24x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mla8.c b/src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mla8.c
new file mode 100644
index 0000000..5906f5a
--- /dev/null
+++ b/src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mla8.c
@@ -0,0 +1,279 @@
+// Auto-generated file. Do not edit!
+// Template: src/qs8-dwconv/unipass-neon-mul8.c.in
+// Generator: tools/xngen
+//
+// Copyright 2020 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+
+#include <assert.h>
+
+#include <arm_neon.h>
+
+#include <xnnpack/dwconv.h>
+
+
+void xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8(
+ size_t channels,
+ size_t output_width,
+ const int8_t** input,
+ const void* weights,
+ int8_t* output,
+ size_t input_stride,
+ size_t output_increment,
+ size_t input_offset,
+ const int8_t* zero,
+ const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
+{
+ assert(channels != 0);
+ assert(output_width != 0);
+
+ const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_pre_shift);
+ const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->rndnu_neon.multiplier);
+ const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_post_shift);
+ const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->rndnu_neon.output_zero_point);
+ const int8x8_t voutput_min = vld1_dup_s8(¶ms->rndnu_neon.output_min);
+ const int8x8_t voutput_max = vld1_dup_s8(¶ms->rndnu_neon.output_max);
+ do {
+ const int8_t* i0 = input[0];
+ assert(i0 != NULL);
+ if XNN_UNPREDICTABLE(i0 != zero) {
+ i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
+ }
+ const int8_t* i1 = input[1];
+ assert(i1 != NULL);
+ if XNN_UNPREDICTABLE(i1 != zero) {
+ i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
+ }
+ const int8_t* i2 = input[2];
+ assert(i2 != NULL);
+ if XNN_UNPREDICTABLE(i2 != zero) {
+ i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
+ }
+ const int8_t* i3 = input[3];
+ assert(i3 != NULL);
+ if XNN_UNPREDICTABLE(i3 != zero) {
+ i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
+ }
+ const int8_t* i4 = input[4];
+ assert(i4 != NULL);
+ if XNN_UNPREDICTABLE(i4 != zero) {
+ i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
+ }
+ const int8_t* i5 = input[5];
+ assert(i5 != NULL);
+ if XNN_UNPREDICTABLE(i5 != zero) {
+ i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
+ }
+ const int8_t* i6 = input[6];
+ assert(i6 != NULL);
+ if XNN_UNPREDICTABLE(i6 != zero) {
+ i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
+ }
+ const int8_t* i7 = input[7];
+ assert(i7 != NULL);
+ if XNN_UNPREDICTABLE(i7 != zero) {
+ i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
+ }
+ const int8_t* i8 = input[8];
+ assert(i8 != NULL);
+ if XNN_UNPREDICTABLE(i8 != zero) {
+ i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
+ }
+ input = (const int8_t**) ((uintptr_t) input + input_stride);
+
+ size_t c = channels;
+ const void* w = weights;
+ for (; c >= 8; c -= 8) {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
+ const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
+ const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
+ const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
+ const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
+ const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
+ const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
+ const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
+ const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
+ const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
+ vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
+
+ vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
+ vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, voutput_min);
+
+ vout01234567 = vmin_s8(vout01234567, voutput_max);
+
+ vst1_s8(output, vout01234567); output += 8;
+ }
+ if XNN_UNLIKELY(c != 0) {
+ {
+ int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+ int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
+
+ const int8x8_t vi0x01234567 = vld1_s8(i0);
+ const int8x8_t vk0x01234567 = vld1_s8(w);
+
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+
+ const int8x8_t vi1x01234567 = vld1_s8(i1);
+ const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi2x01234567 = vld1_s8(i2);
+ const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
+
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+
+ const int8x8_t vi3x01234567 = vld1_s8(i3);
+ const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi4x01234567 = vld1_s8(i4);
+ const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
+
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+
+ const int8x8_t vi5x01234567 = vld1_s8(i5);
+ const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi6x01234567 = vld1_s8(i6);
+ const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
+
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+
+ const int8x8_t vi7x01234567 = vld1_s8(i7);
+ const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
+
+ vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+ const int8x8_t vi8x01234567 = vld1_s8(i8);
+ const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
+
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
+
+ vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
+ vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
+
+ vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
+ vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
+
+#if XNN_ARCH_ARM64
+ const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#else
+ const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
+ int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
+#endif
+
+ vout01234567 = vmax_s8(vout01234567, voutput_min);
+ vout01234567 = vmin_s8(vout01234567, voutput_max);
+
+ if (c & 4) {
+ vst1_lane_u32(__builtin_assume_aligned(output, 1), vreinterpret_u32_s8(vout01234567), 0); output += 4;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 4);
+ }
+ if (c & 2) {
+ vst1_lane_u16(__builtin_assume_aligned(output, 1), vreinterpret_u16_s8(vout01234567), 0); output += 2;
+ vout01234567 = vext_s8(vout01234567, vout01234567, 2);
+ }
+ if (c & 1) {
+ vst1_lane_s8(output, vout01234567, 0); output += 1;
+ }
+ }
+ }
+
+ output = (int8_t*) ((uintptr_t) output + output_increment);
+ } while (--output_width != 0);
+}
diff --git a/src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul8.c b/src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul8.c
index 2a6b160..f21feed 100644
--- a/src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul8.c
+++ b/src/qs8-dwconv/gen/up8x9-minmax-rndnu-neon-mul8.c
@@ -93,74 +93,74 @@
const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
@@ -195,66 +195,66 @@
const int8x8_t vi0x01234567 = vld1_s8(i0);
const int8x8_t vk0x01234567 = vld1_s8(w);
- const int16x8_t vprod0x01234567 = vmull_s8(vi0x01234567, vk0x01234567);
+ int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod0x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod0x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi1x01234567 = vld1_s8(i1);
const int8x8_t vk1x01234567 = vld1_s8((const void*) ((const int8_t*) w + 8));
- const int16x8_t vprod1x01234567 = vmull_s8(vi1x01234567, vk1x01234567);
+ vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod1x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod1x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi2x01234567 = vld1_s8(i2);
const int8x8_t vk2x01234567 = vld1_s8((const void*) ((const int8_t*) w + 16));
- const int16x8_t vprod2x01234567 = vmull_s8(vi2x01234567, vk2x01234567);
+ vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod2x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod2x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi3x01234567 = vld1_s8(i3);
const int8x8_t vk3x01234567 = vld1_s8((const void*) ((const int8_t*) w + 24));
- const int16x8_t vprod3x01234567 = vmull_s8(vi3x01234567, vk3x01234567);
+ vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod3x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod3x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi4x01234567 = vld1_s8(i4);
const int8x8_t vk4x01234567 = vld1_s8((const void*) ((const int8_t*) w + 32));
- const int16x8_t vprod4x01234567 = vmull_s8(vi4x01234567, vk4x01234567);
+ vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod4x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod4x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi5x01234567 = vld1_s8(i5);
const int8x8_t vk5x01234567 = vld1_s8((const void*) ((const int8_t*) w + 40));
- const int16x8_t vprod5x01234567 = vmull_s8(vi5x01234567, vk5x01234567);
+ vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod5x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod5x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi6x01234567 = vld1_s8(i6);
const int8x8_t vk6x01234567 = vld1_s8((const void*) ((const int8_t*) w + 48));
- const int16x8_t vprod6x01234567 = vmull_s8(vi6x01234567, vk6x01234567);
+ vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod6x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod6x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi7x01234567 = vld1_s8(i7);
const int8x8_t vk7x01234567 = vld1_s8((const void*) ((const int8_t*) w + 56));
- const int16x8_t vprod7x01234567 = vmull_s8(vi7x01234567, vk7x01234567);
+ vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod7x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod7x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
const int8x8_t vi8x01234567 = vld1_s8(i8);
const int8x8_t vk8x01234567 = vld1_s8((const void*) ((const int8_t*) w + 64));
- const int16x8_t vprod8x01234567 = vmull_s8(vi8x01234567, vk8x01234567);
+ vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
- vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod8x01234567));
- vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod8x01234567));
+ vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
+ vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
diff --git a/src/qs8-dwconv/unipass-neon-mul8.c.in b/src/qs8-dwconv/unipass-neon-mul8.c.in
index 4c864c4..f88eb05 100644
--- a/src/qs8-dwconv/unipass-neon-mul8.c.in
+++ b/src/qs8-dwconv/unipass-neon-mul8.c.in
@@ -24,7 +24,7 @@
$if REQUANTIZATION == "FP32" and DATATYPE == "QC8" and not ARMV8:
$PARAMS_STRUCT = "neon_fp32"
$ISA = "neonv8" if ARMV8 else "neon"
-void xnn_${DATATYPE.lower()}_dwconv_minmax_${REQUANTIZATION.lower()}_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__${ISA}_mul8(
+void xnn_${DATATYPE.lower()}_dwconv_minmax_${REQUANTIZATION.lower()}_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__${ISA}_${"mla" if MLA else "mul"}8(
size_t channels,
size_t output_width,
const int8_t** input,
@@ -84,12 +84,20 @@
const int8x8_t vi${K}x${ABC[C:C+8]} = vld1_s8(i${K}); i${K} += 8;
const int8x8_t vk${K}x${ABC[C:C+8]} = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
- $for C in range(0, CHANNEL_TILE, 8):
- const int16x8_t vprod${K}x${ABC[C:C+8]} = vmull_s8(vi${K}x${ABC[C:C+8]}, vk${K}x${ABC[C:C+8]});
+ $if K == 0:
+ $for C in range(0, CHANNEL_TILE, 8):
+ int16x8_t vprod${ABC[C:C+8]} = vmull_s8(vi${K}x${ABC[C:C+8]}, vk${K}x${ABC[C:C+8]});
+ $elif K % 2 == 0 or K + 1 == KERNEL_TILE or not MLA:
+ $for C in range(0, CHANNEL_TILE, 8):
+ vprod${ABC[C:C+8]} = vmull_s8(vi${K}x${ABC[C:C+8]}, vk${K}x${ABC[C:C+8]});
+ $else:
+ $for C in range(0, CHANNEL_TILE, 8):
+ vprod${ABC[C:C+8]} = vmlal_s8(vprod${ABC[C:C+8]}, vi${K}x${ABC[C:C+8]}, vk${K}x${ABC[C:C+8]});
- $for C in range(0, CHANNEL_TILE, 8):
- vacc${ABC[C:C+4]} = vaddw_s16(vacc${ABC[C:C+4]}, vget_low_s16(vprod${K}x${ABC[C:C+8]}));
- vacc${ABC[C+4:C+8]} = vaddw_s16(vacc${ABC[C+4:C+8]}, vget_high_s16(vprod${K}x${ABC[C:C+8]}));
+ $if not MLA or K % 2 == 1 or K + 1 == KERNEL_TILE:
+ $for C in range(0, CHANNEL_TILE, 8):
+ vacc${ABC[C:C+4]} = vaddw_s16(vacc${ABC[C:C+4]}, vget_low_s16(vprod${ABC[C:C+8]}));
+ vacc${ABC[C+4:C+8]} = vaddw_s16(vacc${ABC[C+4:C+8]}, vget_high_s16(vprod${ABC[C:C+8]}));
$if REQUANTIZATION == "GEMMLOWP":
$for C in range(0, CHANNEL_TILE, 4):
@@ -227,10 +235,16 @@
$else:
const int8x8_t vk${K}x${ABC[0:8]} = vld1_s8((const void*) ((const int8_t*) w + ${K * CHANNEL_TILE}));
- const int16x8_t vprod${K}x${ABC[0:8]} = vmull_s8(vi${K}x${ABC[0:8]}, vk${K}x${ABC[0:8]});
+ $if K == 0:
+ int16x8_t vprod${ABC[0:8]} = vmull_s8(vi${K}x${ABC[0:8]}, vk${K}x${ABC[0:8]});
+ $elif K % 2 == 0 or K + 1 == KERNEL_TILE or not MLA:
+ vprod${ABC[0:8]} = vmull_s8(vi${K}x${ABC[0:8]}, vk${K}x${ABC[0:8]});
+ $else:
+ vprod${ABC[0:8]} = vmlal_s8(vprod${ABC[0:8]}, vi${K}x${ABC[0:8]}, vk${K}x${ABC[0:8]});
- vacc${ABC[0:4]} = vaddw_s16(vacc${ABC[0:4]}, vget_low_s16(vprod${K}x${ABC[0:8]}));
- vacc${ABC[4:8]} = vaddw_s16(vacc${ABC[4:8]}, vget_high_s16(vprod${K}x${ABC[0:8]}));
+ $if not MLA or K % 2 == 1 or K + 1 == KERNEL_TILE:
+ vacc${ABC[0:4]} = vaddw_s16(vacc${ABC[0:4]}, vget_low_s16(vprod${ABC[0:8]}));
+ vacc${ABC[4:8]} = vaddw_s16(vacc${ABC[4:8]}, vget_high_s16(vprod${ABC[0:8]}));
$if REQUANTIZATION == "GEMMLOWP":
vacc${ABC[0:4]} = vqrdmulhq_s32(vacc${ABC[0:4]}, vmultiplier);
diff --git a/src/xnnpack/dwconv.h b/src/xnnpack/dwconv.h
index d0bdbe9..4d7f862 100644
--- a/src/xnnpack/dwconv.h
+++ b/src/xnnpack/dwconv.h
@@ -391,9 +391,15 @@
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mul8)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mul8)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8)
+
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mul8)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mul8)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8)
+DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8)
+
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up8x9__neon_mul16)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up16x9__neon_mul16)
DECLARE_QS8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qs8_dwconv_minmax_fp32_ukernel_up24x9__neon_mul16)
@@ -635,15 +641,27 @@
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mul8)
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mul8)
+DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8)
+DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8)
+
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mul8)
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mul8)
+DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8)
+DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8)
+
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mul8)
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mul8)
+DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8)
+DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8)
+
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mul8)
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mul8)
+DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8)
+DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8)
+
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mul16)
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mul16)
DECLARE_QC8_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_qc8_dwconv_minmax_fp32_ukernel_up24x9__neon_mul16)
diff --git a/test/qc8-dwconv-minmax-fp32.cc b/test/qc8-dwconv-minmax-fp32.cc
index d4287f8..5d770c0 100644
--- a/test/qc8-dwconv-minmax-fp32.cc
+++ b/test/qc8-dwconv-minmax-fp32.cc
@@ -765,6 +765,750 @@
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, c_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(8)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, c_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, c_div_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, c_div_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, c_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 1; channels < 8; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, c_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, c_gt_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, c_gt_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ for (size_t step = 2; step <= 9; step++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(8)
+ .width(5)
+ .output_stride(43)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .input_offset(176)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 9; mz++) {
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .input_offset(176)
+ .zero_index(mz)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, c_eq_16) {
+ TEST_REQUIRES_ARM_NEON;
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(16)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, c_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, c_div_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, c_div_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, c_lt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 1; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, c_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, c_gt_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, c_gt_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ for (size_t step = 2; step <= 9; step++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(16)
+ .width(5)
+ .output_stride(83)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .input_offset(304)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEON_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 9; mz++) {
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .input_offset(304)
+ .zero_index(mz)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, c_eq_8) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(8)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, c_div_8) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, c_div_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, c_div_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, c_lt_8) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 1; channels < 8; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, c_gt_8) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, c_gt_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, c_gt_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ for (size_t step = 2; step <= 9; step++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(8)
+ .width(5)
+ .output_stride(43)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .input_offset(176)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEONV8_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t mz = 0; mz < 9; mz++) {
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .input_offset(176)
+ .zero_index(mz)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, c_eq_16) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(16)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, c_div_16) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, c_div_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, c_div_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, c_lt_16) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 1; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, c_gt_16) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, c_gt_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, c_gt_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ for (size_t step = 2; step <= 9; step++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(16)
+ .width(5)
+ .output_stride(83)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .input_offset(304)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X9__NEONV8_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t mz = 0; mz < 9; mz++) {
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .input_offset(304)
+ .zero_index(mz)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
TEST(QC8_DWCONV_MINMAX_FP32_UP8X9__NEON_MUL16, c_eq_8) {
TEST_REQUIRES_ARM_NEON;
DWConvMicrokernelTester()
@@ -9270,6 +10014,750 @@
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, c_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(8)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, c_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, c_div_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, c_div_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, c_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 1; channels < 8; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, c_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, c_gt_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, c_gt_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ for (size_t step = 2; step <= 25; step++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(8)
+ .width(5)
+ .output_stride(43)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .input_offset(176)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 25; mz++) {
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .input_offset(176)
+ .zero_index(mz)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, c_eq_16) {
+ TEST_REQUIRES_ARM_NEON;
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(16)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, c_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, c_div_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, c_div_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, c_lt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 1; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, c_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, c_gt_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, c_gt_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ for (size_t step = 2; step <= 25; step++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(16)
+ .width(5)
+ .output_stride(83)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .input_offset(304)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEON_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 25; mz++) {
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .input_offset(304)
+ .zero_index(mz)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8, xnn_init_qs8_minmax_neon_fp32_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, c_eq_8) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(8)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, c_div_8) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, c_div_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, c_div_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, c_lt_8) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 1; channels < 8; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, c_gt_8) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, c_gt_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, c_gt_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ for (size_t step = 2; step <= 25; step++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(8)
+ .width(5)
+ .output_stride(43)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .input_offset(176)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEONV8_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t mz = 0; mz < 25; mz++) {
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .input_offset(176)
+ .zero_index(mz)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, c_eq_16) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(16)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, c_div_16) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, c_div_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, c_div_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, c_lt_16) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 1; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, c_gt_16) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, c_gt_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, c_gt_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ for (size_t step = 2; step <= 25; step++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(16)
+ .width(5)
+ .output_stride(83)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .input_offset(304)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+
+ TEST(QC8_DWCONV_MINMAX_FP32_UP16X25__NEONV8_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON_V8;
+ for (uint32_t mz = 0; mz < 25; mz++) {
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .input_offset(304)
+ .zero_index(mz)
+ .Test(xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8, xnn_init_qs8_minmax_neon_params, xnn_init_qs8_requantization_fp32_params, xnn_qs8_requantize_fp32);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
TEST(QC8_DWCONV_MINMAX_FP32_UP8X25__NEON_MUL16, c_eq_8) {
TEST_REQUIRES_ARM_NEON;
DWConvMicrokernelTester()
diff --git a/test/qc8-dwconv-minmax-fp32.yaml b/test/qc8-dwconv-minmax-fp32.yaml
index 6687498..cc46f12 100644
--- a/test/qc8-dwconv-minmax-fp32.yaml
+++ b/test/qc8-dwconv-minmax-fp32.yaml
@@ -11,6 +11,14 @@
init: xnn_init_qs8_minmax_neon_params
- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mul8
init: xnn_init_qs8_minmax_neon_params
+- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mla8
+ init: xnn_init_qs8_minmax_neon_fp32_params
+- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mla8
+ init: xnn_init_qs8_minmax_neon_fp32_params
+- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neonv8_mla8
+ init: xnn_init_qs8_minmax_neon_params
+- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neonv8_mla8
+ init: xnn_init_qs8_minmax_neon_params
- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up8x9__neon_mul16
init: xnn_init_qs8_minmax_neon_fp32_params
- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up16x9__neon_mul16
@@ -105,6 +113,14 @@
init: xnn_init_qs8_minmax_neon_params
- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mul8
init: xnn_init_qs8_minmax_neon_params
+- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mla8
+ init: xnn_init_qs8_minmax_neon_fp32_params
+- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mla8
+ init: xnn_init_qs8_minmax_neon_fp32_params
+- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neonv8_mla8
+ init: xnn_init_qs8_minmax_neon_params
+- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neonv8_mla8
+ init: xnn_init_qs8_minmax_neon_params
- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up8x25__neon_mul16
init: xnn_init_qs8_minmax_neon_fp32_params
- name: xnn_qc8_dwconv_minmax_fp32_ukernel_up16x25__neon_mul16
diff --git a/test/qs8-dwconv-minmax-rndnu.cc b/test/qs8-dwconv-minmax-rndnu.cc
index 8d75072..2ab8afa 100644
--- a/test/qs8-dwconv-minmax-rndnu.cc
+++ b/test/qs8-dwconv-minmax-rndnu.cc
@@ -765,6 +765,750 @@
#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, c_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(8)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, c_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, c_div_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, c_div_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, c_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 1; channels < 8; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, c_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, c_gt_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, c_gt_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ for (size_t step = 2; step <= 9; step++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(8)
+ .width(5)
+ .output_stride(43)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .input_offset(176)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 9; mz++) {
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(9)
+ .channels(channels)
+ .input_offset(176)
+ .zero_index(mz)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, c_eq_16) {
+ TEST_REQUIRES_ARM_NEON;
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(16)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, c_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, c_div_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, c_div_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, c_lt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 1; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, c_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, c_gt_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, c_gt_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ for (size_t step = 2; step <= 9; step++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(16)
+ .width(5)
+ .output_stride(83)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .input_offset(304)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X9__NEON_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 9; mz++) {
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(9)
+ .channels(channels)
+ .input_offset(304)
+ .zero_index(mz)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, c_eq_8) {
+ TEST_REQUIRES_ARM_NEON;
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(8)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, c_div_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, c_div_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, c_div_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, c_lt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 1; channels < 8; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, c_gt_8) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, c_gt_8_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, c_gt_8_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 9; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ for (size_t step = 2; step <= 25; step++) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(8)
+ .width(5)
+ .output_stride(43)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 40; channels += 7) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .input_offset(176)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X25__NEON_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 25; mz++) {
+ for (uint32_t channels = 16; channels < 128; channels += 24) {
+ DWConvMicrokernelTester()
+ .cr(8)
+ .kr(25)
+ .channels(channels)
+ .input_offset(176)
+ .zero_index(mz)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, c_eq_16) {
+ TEST_REQUIRES_ARM_NEON;
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(16)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, c_div_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, c_div_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, c_div_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, c_lt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 1; channels < 16; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, c_gt_16) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, c_gt_16_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, c_gt_16_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 17; channels < 32; channels++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, multipixel) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, multipixel_with_step) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ for (size_t step = 2; step <= 25; step++) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .step(step)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, multipixel_with_output_stride) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(16)
+ .width(5)
+ .output_stride(83)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, multipixel_with_qmin) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmin(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, multipixel_with_qmax) {
+ TEST_REQUIRES_ARM_NEON;
+ for (size_t channels = 1; channels <= 80; channels += 15) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .width(3)
+ .qmax(128)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, input_offset) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .input_offset(304)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+
+ TEST(QS8_DWCONV_MINMAX_RNDNU_UP16X25__NEON_MLA8, zero) {
+ TEST_REQUIRES_ARM_NEON;
+ for (uint32_t mz = 0; mz < 25; mz++) {
+ for (uint32_t channels = 32; channels < 256; channels += 48) {
+ DWConvMicrokernelTester()
+ .cr(16)
+ .kr(25)
+ .channels(channels)
+ .input_offset(304)
+ .zero_index(mz)
+ .Test(xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8, xnn_init_qs8_conv_minmax_rndnu_neon_params, xnn_init_qs8_requantization_rndnu_params, xnn_qs8_requantize_rndnu);
+ }
+ }
+ }
+#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64
+
+
+#if XNN_ARCH_ARM || XNN_ARCH_ARM64
TEST(QS8_DWCONV_MINMAX_RNDNU_UP8X9__NEON_MUL16, c_eq_8) {
TEST_REQUIRES_ARM_NEON;
DWConvMicrokernelTester()
diff --git a/test/qs8-dwconv-minmax-rndnu.yaml b/test/qs8-dwconv-minmax-rndnu.yaml
index 0e8b762..75a8547 100644
--- a/test/qs8-dwconv-minmax-rndnu.yaml
+++ b/test/qs8-dwconv-minmax-rndnu.yaml
@@ -10,6 +10,14 @@
init: xnn_init_qs8_conv_minmax_rndnu_neon_params
- name: xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mul8
init: xnn_init_qs8_conv_minmax_rndnu_neon_params
+- name: xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mla8
+ init: xnn_init_qs8_conv_minmax_rndnu_neon_params
+- name: xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mla8
+ init: xnn_init_qs8_conv_minmax_rndnu_neon_params
+- name: xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mla8
+ init: xnn_init_qs8_conv_minmax_rndnu_neon_params
+- name: xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mla8
+ init: xnn_init_qs8_conv_minmax_rndnu_neon_params
- name: xnn_qs8_dwconv_minmax_rndnu_ukernel_up8x9__neon_mul16
init: xnn_init_qs8_conv_minmax_rndnu_neon_params
- name: xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x9__neon_mul16
