Relax the tolerable range for quant and boolean values.
Before this CL, the quant tolerable range is too strict for complex
operations such as HEATMAP_MAX_KEYPOINT. The absolute tolerance, bias,
and MSE criteria for quantized tensors are slightly relaxed.
Before this CL, the accuracy checker does not allow any boolean value
mismatch. However, there are several cases that two floating point values
are very close to each other, and the result of comparison operations, e.g.
GREATER, ends up to be flipped because of the accumulated error. With
this CL, we only expect the number of mismatches does not exceed a
certain ratio.
Bug: 134801089
Test: NNT_static_fuzzing
Change-Id: I7faabafce91b245f525b4ef39736862d82f38edc
diff --git a/runtime/test/fuzzing/RandomGraphGenerator.cpp b/runtime/test/fuzzing/RandomGraphGenerator.cpp
index 96ac5d1..6c21b7e 100644
--- a/runtime/test/fuzzing/RandomGraphGenerator.cpp
+++ b/runtime/test/fuzzing/RandomGraphGenerator.cpp
@@ -320,17 +320,26 @@
EXPECT_LE(mse, criterion.mse);
}
-void expectBooleanEqual(const RandomOperand& op, const OperandBuffer& test) {
+// For boolean values, we expect the number of mismatches does not exceed a certain ratio.
+void expectBooleanNearlyEqual(const RandomOperand& op, const OperandBuffer& test,
+ float allowedErrorRatio) {
const bool8* actual = reinterpret_cast<const bool8*>(test.data());
const bool8* expected = reinterpret_cast<const bool8*>(op.buffer.data());
uint32_t len = op.getNumberOfElements();
uint32_t numErrors = 0;
+ std::stringstream errorMsg;
for (uint32_t i = 0; i < len; i++) {
- SCOPED_TRACE(testing::Message() << "When comparing element " << i);
- if (numErrors < kMaxNumberOfPrintedErrors) EXPECT_EQ(expected[i], actual[i]);
- if (expected[i] != actual[i]) numErrors++;
+ if (expected[i] != actual[i]) {
+ if (numErrors < kMaxNumberOfPrintedErrors)
+ errorMsg << " Expected: " << expected[i] << ", actual: " << actual[i]
+ << ", when comparing element " << i << "\n";
+ numErrors++;
+ }
}
- EXPECT_EQ(numErrors, 0u);
+ // When |len| is small, the allowedErrorCount will intentionally ceil at 1, which allows for
+ // greater tolerance.
+ uint32_t allowedErrorCount = static_cast<uint32_t>(std::ceil(allowedErrorRatio * len));
+ EXPECT_LE(numErrors, allowedErrorCount) << errorMsg.str();
}
void RandomGraph::checkResults(const std::vector<OperandBuffer>& buffers,
@@ -367,7 +376,7 @@
expectNear<int16_t>(*op, buffers[i], criteria.quant16Symm);
break;
case Type::TENSOR_BOOL8:
- expectBooleanEqual(*op, buffers[i]);
+ expectBooleanNearlyEqual(*op, buffers[i], /*allowedErrorRatio=*/0.01);
break;
default:
NN_FUZZER_CHECK(false) << "Data type not supported.";
diff --git a/runtime/test/fuzzing/TestRandomGraph.cpp b/runtime/test/fuzzing/TestRandomGraph.cpp
index eb6d4c6..4135bc6 100644
--- a/runtime/test/fuzzing/TestRandomGraph.cpp
+++ b/runtime/test/fuzzing/TestRandomGraph.cpp
@@ -413,10 +413,10 @@
.float32 = {.atol = 1e-3f, .rtol = 1e-3f, .bias = 2e-5f, .mse = 1e-7f},
.float16 = {.atol = 1.0f, .rtol = 1.0f, .bias = 5e-3f, .mse = 1e-4f},
.int32 = {.atol = 1},
- .quant8Asymm = {.atol = 8, .bias = 1, .mse = 1},
- .quant8Symm = {.atol = 8, .bias = 1, .mse = 1},
- .quant16Asymm = {.atol = 8, .bias = 1, .mse = 1},
- .quant16Symm = {.atol = 8, .bias = 1, .mse = 1}};
+ .quant8Asymm = {.atol = 10, .bias = 1.5, .mse = 1.5},
+ .quant8Symm = {.atol = 10, .bias = 1.5, .mse = 1.5},
+ .quant16Asymm = {.atol = 10, .bias = 1.5, .mse = 1.5},
+ .quant16Symm = {.atol = 10, .bias = 1.5, .mse = 1.5}};
/*-- NNAPI 1.0 Operations ---------------------------------------------------*/
@@ -565,19 +565,19 @@
.float32 = {.atol = 1e-2f, .rtol = 1e-2f, .bias = 2e-5f, .mse = 1e-7f},
.float16 = {.atol = 1.0f, .rtol = 1.0f, .bias = 5e-3f, .mse = 1e-4f},
.int32 = {.atol = 1},
- .quant8Asymm = {.atol = 8, .bias = 1, .mse = 1},
- .quant8Symm = {.atol = 8, .bias = 1, .mse = 1},
- .quant16Asymm = {.atol = 8, .bias = 1, .mse = 1},
- .quant16Symm = {.atol = 8, .bias = 1, .mse = 1}};
+ .quant8Asymm = {.atol = 12, .bias = 2, .mse = 2},
+ .quant8Symm = {.atol = 12, .bias = 2, .mse = 2},
+ .quant16Asymm = {.atol = 12, .bias = 2, .mse = 2},
+ .quant16Symm = {.atol = 12, .bias = 2, .mse = 2}};
const AccuracyCriteria kLargeGraphCriteria = {
.float32 = {.atol = 1e-1f, .rtol = 1e-1f, .bias = 1e-2f, .mse = 1e-4f},
.float16 = {.atol = 1.0f, .rtol = 1.0f, .bias = 1e-1f, .mse = 5e-2f},
.int32 = {.atol = 1},
- .quant8Asymm = {.atol = 10, .bias = 2, .mse = 2},
- .quant8Symm = {.atol = 10, .bias = 2, .mse = 2},
- .quant16Asymm = {.atol = 10, .bias = 2, .mse = 2},
- .quant16Symm = {.atol = 10, .bias = 2, .mse = 2}};
+ .quant8Asymm = {.atol = 12, .bias = 2, .mse = 2},
+ .quant8Symm = {.atol = 12, .bias = 2, .mse = 2},
+ .quant16Asymm = {.atol = 12, .bias = 2, .mse = 2},
+ .quant16Symm = {.atol = 12, .bias = 2, .mse = 2}};
// Due to the limitation of the random graph generator, graphs generated with mixed-type or
// mixed-rank operations are likely to result in a disconnected network. Thus, we filter the
