Consolidate requantization structures and functions in a single header
PiperOrigin-RevId: 375642230
diff --git a/src/xnnpack/params-init.h b/src/xnnpack/params-init.h
index 2448b21..a245112 100644
--- a/src/xnnpack/params-init.h
+++ b/src/xnnpack/params-init.h
@@ -1547,69 +1547,3 @@
params->scalar.output_min = (int32_t) output_min;
params->scalar.output_max = (int32_t) output_max;
}
-
-static inline void xnn_init_scalar_qu8_requantization_params(
- union xnn_qu8_requantization_params params[XNN_MIN_ELEMENTS(1)],
- float scale,
- uint8_t zero_point,
- uint8_t min,
- uint8_t max)
-{
- // Compute requantization parameters.
- assert(scale < 1.0f);
- assert(scale >= 0x1.0p-32f);
- const uint32_t scale_bits = fp32_to_bits(scale);
-
- // Multiplier is in [0x40000000, 0x7FFFFF80] range.
- const int32_t multiplier = (int32_t)(((scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7);
- assert(multiplier >= INT32_C(0x40000000));
- assert(multiplier <= INT32_C(0x7FFFFF80));
-
- // Shift is in [0, 31] range.
- const int32_t shift = 127 + 31 - 32 - (fp32_to_bits(scale) >> 23);
- assert(shift >= 0);
- assert(shift < 32);
-
- const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
- const uint32_t remainder_threshold = remainder_mask >> 1;
- params->gemmlowp.multiplier = multiplier;
- params->gemmlowp.remainder_mask = (int32_t) remainder_mask;
- params->gemmlowp.remainder_threshold = (int32_t) remainder_threshold;
- params->gemmlowp.shift = (uint32_t) shift;
- params->gemmlowp.min_less_zero_point = (int32_t) (uint32_t) min - (int32_t) (uint32_t) zero_point;
- params->gemmlowp.max_less_zero_point = (int32_t) (uint32_t) max - (int32_t) (uint32_t) zero_point;
- params->gemmlowp.zero_point = (int32_t) (uint32_t) zero_point;
-}
-
-static inline void xnn_init_scalar_qs8_requantization_params(
- union xnn_qs8_requantization_params params[XNN_MIN_ELEMENTS(1)],
- float scale,
- int8_t zero_point,
- int8_t min,
- int8_t max)
-{
- // Compute requantization parameters.
- assert(scale < 1.0f);
- assert(scale >= 0x1.0p-32f);
- const uint32_t scale_bits = fp32_to_bits(scale);
-
- // Multiplier is in [0x40000000, 0x7FFFFF80] range.
- const int32_t multiplier = (int32_t)(((scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7);
- assert(multiplier >= INT32_C(0x40000000));
- assert(multiplier <= INT32_C(0x7FFFFF80));
-
- // Shift is in [0, 31] range.
- const int32_t shift = 127 + 31 - 32 - (fp32_to_bits(scale) >> 23);
- assert(shift >= 0);
- assert(shift < 32);
-
- const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
- const uint32_t remainder_threshold = remainder_mask >> 1;
- params->gemmlowp.multiplier = multiplier;
- params->gemmlowp.remainder_mask = (int32_t) remainder_mask;
- params->gemmlowp.remainder_threshold = (int32_t) remainder_threshold;
- params->gemmlowp.shift = (uint32_t) shift;
- params->gemmlowp.min_less_zero_point = (int32_t) min - (int32_t) zero_point;
- params->gemmlowp.max_less_zero_point = (int32_t) max - (int32_t) zero_point;
- params->gemmlowp.zero_point = (int32_t) zero_point;
-}
diff --git a/src/xnnpack/params.h b/src/xnnpack/params.h
index 1fcf76a..54955a4 100644
--- a/src/xnnpack/params.h
+++ b/src/xnnpack/params.h
@@ -523,30 +523,6 @@
#endif // XNN_ARCH_WASMSIMD
};
-union xnn_qu8_requantization_params {
- struct {
- int32_t multiplier;
- int32_t remainder_mask;
- int32_t remainder_threshold;
- uint32_t shift;
- int32_t min_less_zero_point;
- int32_t max_less_zero_point;
- int32_t zero_point;
- } gemmlowp;
-};
-
-union xnn_qs8_requantization_params {
- struct {
- int32_t multiplier;
- int32_t remainder_mask;
- int32_t remainder_threshold;
- uint32_t shift;
- int32_t min_less_zero_point;
- int32_t max_less_zero_point;
- int32_t zero_point;
- } gemmlowp;
-};
-
typedef void (*xnn_ppmm_ukernel_function)(
size_t mr,
size_t nc,
diff --git a/src/xnnpack/requantization.h b/src/xnnpack/requantization.h
index 7afa01e..7990e2c 100644
--- a/src/xnnpack/requantization.h
+++ b/src/xnnpack/requantization.h
@@ -20,6 +20,51 @@
#include <xnnpack/params.h>
+union xnn_qu8_requantization_params {
+ struct {
+ int32_t multiplier;
+ int32_t remainder_mask;
+ int32_t remainder_threshold;
+ uint32_t shift;
+ int32_t min_less_zero_point;
+ int32_t max_less_zero_point;
+ int32_t zero_point;
+ } gemmlowp;
+};
+
+static inline void xnn_init_qu8_requantization_gemmlowp_params(
+ union xnn_qu8_requantization_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ uint8_t zero_point,
+ uint8_t min,
+ uint8_t max)
+{
+ // Compute requantization parameters.
+ assert(scale < 1.0f);
+ assert(scale >= 0x1.0p-32f);
+ const uint32_t scale_bits = fp32_to_bits(scale);
+
+ // Multiplier is in [0x40000000, 0x7FFFFF80] range.
+ const int32_t multiplier = (int32_t)(((scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7);
+ assert(multiplier >= INT32_C(0x40000000));
+ assert(multiplier <= INT32_C(0x7FFFFF80));
+
+ // Shift is in [0, 31] range.
+ const int32_t shift = 127 + 31 - 32 - (fp32_to_bits(scale) >> 23);
+ assert(shift >= 0);
+ assert(shift < 32);
+
+ const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
+ const uint32_t remainder_threshold = remainder_mask >> 1;
+ params->gemmlowp.multiplier = multiplier;
+ params->gemmlowp.remainder_mask = (int32_t) remainder_mask;
+ params->gemmlowp.remainder_threshold = (int32_t) remainder_threshold;
+ params->gemmlowp.shift = (uint32_t) shift;
+ params->gemmlowp.min_less_zero_point = (int32_t) (uint32_t) min - (int32_t) (uint32_t) zero_point;
+ params->gemmlowp.max_less_zero_point = (int32_t) (uint32_t) max - (int32_t) (uint32_t) zero_point;
+ params->gemmlowp.zero_point = (int32_t) (uint32_t) zero_point;
+}
+
static inline uint8_t xnn_qu8_requantize_gemmlowp(
int32_t n,
union xnn_qu8_requantization_params params)
@@ -33,6 +78,52 @@
return (uint8_t) (n + params.gemmlowp.zero_point);
}
+
+union xnn_qs8_requantization_params {
+ struct {
+ int32_t multiplier;
+ int32_t remainder_mask;
+ int32_t remainder_threshold;
+ uint32_t shift;
+ int32_t min_less_zero_point;
+ int32_t max_less_zero_point;
+ int32_t zero_point;
+ } gemmlowp;
+};
+
+static inline void xnn_init_qs8_requantization_gemmlowp_params(
+ union xnn_qs8_requantization_params params[XNN_MIN_ELEMENTS(1)],
+ float scale,
+ int8_t zero_point,
+ int8_t min,
+ int8_t max)
+{
+ // Compute requantization parameters.
+ assert(scale < 1.0f);
+ assert(scale >= 0x1.0p-32f);
+ const uint32_t scale_bits = fp32_to_bits(scale);
+
+ // Multiplier is in [0x40000000, 0x7FFFFF80] range.
+ const int32_t multiplier = (int32_t)(((scale_bits & UINT32_C(0x007FFFFF)) | UINT32_C(0x00800000)) << 7);
+ assert(multiplier >= INT32_C(0x40000000));
+ assert(multiplier <= INT32_C(0x7FFFFF80));
+
+ // Shift is in [0, 31] range.
+ const int32_t shift = 127 + 31 - 32 - (fp32_to_bits(scale) >> 23);
+ assert(shift >= 0);
+ assert(shift < 32);
+
+ const uint32_t remainder_mask = (UINT32_C(1) << shift) - UINT32_C(1);
+ const uint32_t remainder_threshold = remainder_mask >> 1;
+ params->gemmlowp.multiplier = multiplier;
+ params->gemmlowp.remainder_mask = (int32_t) remainder_mask;
+ params->gemmlowp.remainder_threshold = (int32_t) remainder_threshold;
+ params->gemmlowp.shift = (uint32_t) shift;
+ params->gemmlowp.min_less_zero_point = (int32_t) min - (int32_t) zero_point;
+ params->gemmlowp.max_less_zero_point = (int32_t) max - (int32_t) zero_point;
+ params->gemmlowp.zero_point = (int32_t) zero_point;
+}
+
static inline int8_t xnn_qs8_requantize_gemmlowp(
int32_t n,
union xnn_qs8_requantization_params params)
diff --git a/test/dwconv-microkernel-tester.h b/test/dwconv-microkernel-tester.h
index f5aff33..e4f0f69 100644
--- a/test/dwconv-microkernel-tester.h
+++ b/test/dwconv-microkernel-tester.h
@@ -235,14 +235,14 @@
union xnn_qu8_conv_minmax_params quantization_params;
init_params(&quantization_params,
kernel_zero_point(), requantization_scale, output_zero_point, qmin(), qmax());
- union xnn_qu8_requantization_params scalar_requantization_params;
- xnn_init_scalar_qu8_requantization_params(
- &scalar_requantization_params, requantization_scale, output_zero_point, qmin(), qmax());
+ union xnn_qu8_requantization_params requantization_params;
+ xnn_init_qu8_requantization_gemmlowp_params(&requantization_params,
+ requantization_scale, output_zero_point, qmin(), qmax());
// Renormalize reference results.
for (size_t x = 0; x < width(); x++) {
for (size_t c = 0; c < channels(); c++) {
- output_ref[x * channels() + c] = xnn_qu8_requantize_gemmlowp(accumulators[x * channels() + c], scalar_requantization_params);
+ output_ref[x * channels() + c] = xnn_qu8_requantize_gemmlowp(accumulators[x * channels() + c], requantization_params);
}
}
@@ -341,14 +341,14 @@
union xnn_qs8_conv_minmax_params quantization_params;
init_params(&quantization_params,
requantization_scale, output_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
- union xnn_qs8_requantization_params scalar_requantization_params;
- xnn_init_scalar_qs8_requantization_params(
- &scalar_requantization_params, requantization_scale, output_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
+ union xnn_qs8_requantization_params requantization_params;
+ xnn_init_qs8_requantization_gemmlowp_params(&requantization_params,
+ requantization_scale, output_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
// Renormalize reference results.
for (size_t x = 0; x < width(); x++) {
for (size_t c = 0; c < channels(); c++) {
- output_ref[x * channels() + c] = xnn_qs8_requantize_gemmlowp(accumulators[x * channels() + c], scalar_requantization_params);
+ output_ref[x * channels() + c] = xnn_qs8_requantize_gemmlowp(accumulators[x * channels() + c], requantization_params);
}
}
diff --git a/test/gemm-microkernel-tester.h b/test/gemm-microkernel-tester.h
index 76a02a6..295d29d 100644
--- a/test/gemm-microkernel-tester.h
+++ b/test/gemm-microkernel-tester.h
@@ -271,9 +271,9 @@
union xnn_qu8_conv_minmax_params quantization_params;
init_params(&quantization_params,
b_zero_point(), requantization_scale, c_zero_point, qmin(), qmax());
- union xnn_qu8_requantization_params scalar_requantization_params;
- xnn_init_scalar_qu8_requantization_params(
- &scalar_requantization_params, requantization_scale, c_zero_point, qmin(), qmax());
+ union xnn_qu8_requantization_params requantization_params;
+ xnn_init_qu8_requantization_gemmlowp_params(&requantization_params,
+ requantization_scale, c_zero_point, qmin(), qmax());
gemm(
m(), n(), k(),
@@ -284,7 +284,7 @@
for (size_t m_index = 0; m_index < m(); m_index++) {
for (size_t n_index = 0; n_index < n(); n_index++) {
- c_ref[m_index * n() + n_index] = xnn_qu8_requantize_gemmlowp(acc[m_index * n() + n_index], scalar_requantization_params);
+ c_ref[m_index * n() + n_index] = xnn_qu8_requantize_gemmlowp(acc[m_index * n() + n_index], requantization_params);
}
}
@@ -389,9 +389,9 @@
union xnn_qu8_conv_minmax_params quantization_params;
init_params(&quantization_params,
b_zero_point(), requantization_scale, c_zero_point, qmin(), qmax());
- union xnn_qu8_requantization_params scalar_requantization_params;
- xnn_init_scalar_qu8_requantization_params(
- &scalar_requantization_params, requantization_scale, c_zero_point, qmin(), qmax());
+ union xnn_qu8_requantization_params requantization_params;
+ xnn_init_qu8_requantization_gemmlowp_params(&requantization_params,
+ requantization_scale, c_zero_point, qmin(), qmax());
const uint8_t* zero_pointer = (zero_index() != SIZE_MAX) ? a.data() : NULL;
@@ -404,7 +404,7 @@
for (size_t m_index = 0; m_index < m(); m_index++) {
for (size_t n_index = 0; n_index < n(); n_index++) {
- c_ref[m_index * n() + n_index] = xnn_qu8_requantize_gemmlowp(acc[m_index * n() + n_index], scalar_requantization_params);
+ c_ref[m_index * n() + n_index] = xnn_qu8_requantize_gemmlowp(acc[m_index * n() + n_index], requantization_params);
}
}
@@ -485,9 +485,9 @@
union xnn_qs8_conv_minmax_params quantization_params;
init_params(&quantization_params,
requantization_scale, c_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
- union xnn_qs8_requantization_params scalar_requantization_params;
- xnn_init_scalar_qs8_requantization_params(
- &scalar_requantization_params, requantization_scale, c_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
+ union xnn_qs8_requantization_params requantization_params;
+ xnn_init_qs8_requantization_gemmlowp_params(&requantization_params,
+ requantization_scale, c_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
gemm(
m(), n(), k(),
@@ -498,7 +498,7 @@
for (size_t m_index = 0; m_index < m(); m_index++) {
for (size_t n_index = 0; n_index < n(); n_index++) {
- c_ref[m_index * n() + n_index] = xnn_qs8_requantize_gemmlowp(acc[m_index * n() + n_index], scalar_requantization_params);
+ c_ref[m_index * n() + n_index] = xnn_qs8_requantize_gemmlowp(acc[m_index * n() + n_index], requantization_params);
}
}
@@ -604,9 +604,9 @@
union xnn_qs8_conv_minmax_params quantization_params;
init_params(&quantization_params,
requantization_scale, c_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
- union xnn_qs8_requantization_params scalar_requantization_params;
- xnn_init_scalar_qs8_requantization_params(
- &scalar_requantization_params, requantization_scale, c_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
+ union xnn_qs8_requantization_params requantization_params;
+ xnn_init_qs8_requantization_gemmlowp_params(&requantization_params,
+ requantization_scale, c_zero_point, int8_t(qmin() - 0x80), int8_t(qmax() - 0x80));
const int8_t* zero_pointer = (zero_index() != SIZE_MAX) ? a.data() : NULL;
@@ -619,7 +619,7 @@
for (size_t m_index = 0; m_index < m(); m_index++) {
for (size_t n_index = 0; n_index < n(); n_index++) {
- c_ref[m_index * n() + n_index] = xnn_qs8_requantize_gemmlowp(acc[m_index * n() + n_index], scalar_requantization_params);
+ c_ref[m_index * n() + n_index] = xnn_qs8_requantize_gemmlowp(acc[m_index * n() + n_index], requantization_params);
}
}
