Add a model loading/unloading stress test am: 814f8a7c82
am: 6cf8fcc31f
Change-Id: I50f4b6241c2797f698edb1d539d7a30833722ea5
diff --git a/build_and_run_benchmark.sh b/build_and_run_benchmark.sh
index 83ca2ab..ab40c49 100755
--- a/build_and_run_benchmark.sh
+++ b/build_and_run_benchmark.sh
@@ -4,18 +4,21 @@
#
# Output is logged to a temporary folder and summarized in txt and JSON formats.
-case "$1" in
- ""|scoring)
- MODE=scoring
+MODE="${1:=scoring}"
+
+case "$MODE" in
+ scoring)
CLASS=com.android.nn.benchmark.app.NNScoringTest
;;
- stress)
- MODE=stress
- CLASS=com.android.nn.benchmark.app.NNStressTest
+ inference-stress)
+ CLASS=com.android.nn.benchmark.app.NNInferenceStressTest
+ ;;
+ model-loading-stress)
+ CLASS=com.android.nn.benchmark.app.NNModelLoadingStressTest
;;
*)
echo "Unknown execution mode: $1"
- echo "Known modes: scoring (default), stress"
+ echo "Known modes: scoring (default), inference-stress, model-loading-stress"
exit 1
;;
esac
diff --git a/jni/run_tflite.cpp b/jni/run_tflite.cpp
index fc722b5..9472480 100644
--- a/jni/run_tflite.cpp
+++ b/jni/run_tflite.cpp
@@ -236,7 +236,9 @@
int seqInferencesMaxCount, float timeout, int flags,
std::vector<InferenceResult>* results) {
if (inOutData.empty()) {
- FATAL("Input/output vector is empty");
+ __android_log_print(ANDROID_LOG_WARN, LOG_TAG,
+ "Input/output vector is empty");
+ return true;
}
float inferenceTotal = 0.0;
diff --git a/src/com/android/nn/benchmark/app/NNStressTest.java b/src/com/android/nn/benchmark/app/NNInferenceStressTest.java
similarity index 80%
rename from src/com/android/nn/benchmark/app/NNStressTest.java
rename to src/com/android/nn/benchmark/app/NNInferenceStressTest.java
index b586417..c4fbdc1 100644
--- a/src/com/android/nn/benchmark/app/NNStressTest.java
+++ b/src/com/android/nn/benchmark/app/NNInferenceStressTest.java
@@ -28,18 +28,18 @@
import org.junit.runners.Parameterized.Parameters;
/**
- * Tests that ensure stability of NNAPI by putting it under heavy load.
+ * Tests that ensure stability of NNAPI by running inference for a
+ * prolonged period of time.
*/
@RunWith(Parameterized.class)
-public class NNStressTest extends BenchmarkTestBase {
- private static final String TAG = NNStressTest.class.getSimpleName();
+public class NNInferenceStressTest extends BenchmarkTestBase {
+ private static final String TAG = NNInferenceStressTest.class.getSimpleName();
private static final String[] MODEL_NAMES = NNScoringTest.MODEL_NAMES;
+ private static final float WARMUP_SECONDS = 0; // No warmup.
+ private static final float RUNTIME_SECONDS = 60 * 60; // 1 hour.
- private static final float STRESS_TEST_WARMUP_SECONDS = 0; // No warmup.
- private static final float STRESS_TEST_RUNTIME_SECONDS = 60 * 60; // 1 hour.
-
- public NNStressTest(TestModels.TestModelEntry model) {
+ public NNInferenceStressTest(TestModels.TestModelEntry model) {
super(model);
}
@@ -68,8 +68,7 @@
public void stressTestNNAPI() throws IOException {
setUseNNApi(true);
setCompleteInputSet(false);
- TestAction ta = new TestAction(mModel, STRESS_TEST_WARMUP_SECONDS,
- STRESS_TEST_RUNTIME_SECONDS);
+ TestAction ta = new TestAction(mModel, WARMUP_SECONDS, RUNTIME_SECONDS);
runTest(ta, mModel.getTestName());
}
}
diff --git a/src/com/android/nn/benchmark/app/NNStressTest.java b/src/com/android/nn/benchmark/app/NNModelLoadingStressTest.java
similarity index 62%
copy from src/com/android/nn/benchmark/app/NNStressTest.java
copy to src/com/android/nn/benchmark/app/NNModelLoadingStressTest.java
index b586417..1ab5e2e 100644
--- a/src/com/android/nn/benchmark/app/NNStressTest.java
+++ b/src/com/android/nn/benchmark/app/NNModelLoadingStressTest.java
@@ -17,29 +17,34 @@
package com.android.nn.benchmark.app;
import android.test.suitebuilder.annotation.LargeTest;
+import android.util.Log;
+import com.android.nn.benchmark.core.InferenceInOutSequence;
import com.android.nn.benchmark.core.TestModels;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.junit.Rule;
import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
+import org.junit.rules.Stopwatch;
import org.junit.runners.Parameterized.Parameters;
/**
- * Tests that ensure stability of NNAPI by putting it under heavy load.
+ * Tests that ensure stability of NNAPI by loading models for a prolonged
+ * period of time.
*/
-@RunWith(Parameterized.class)
-public class NNStressTest extends BenchmarkTestBase {
- private static final String TAG = NNStressTest.class.getSimpleName();
+public class NNModelLoadingStressTest extends BenchmarkTestBase {
+ private static final String TAG = NNModelLoadingStressTest.class.getSimpleName();
private static final String[] MODEL_NAMES = NNScoringTest.MODEL_NAMES;
+ private static final float WARMUP_SECONDS = 0; // No warmup.
+ private static final float INFERENCE_SECONDS = 0; // No inference.
+ private static final float RUNTIME_SECONDS = 30 * 60;
- private static final float STRESS_TEST_WARMUP_SECONDS = 0; // No warmup.
- private static final float STRESS_TEST_RUNTIME_SECONDS = 60 * 60; // 1 hour.
+ @Rule public Stopwatch stopwatch = new Stopwatch() {};
- public NNStressTest(TestModels.TestModelEntry model) {
+ public NNModelLoadingStressTest(TestModels.TestModelEntry model) {
super(model);
}
@@ -53,8 +58,8 @@
model.mModelName,
model.mBaselineSec,
model.mInputShape,
- model.mInOutAssets,
- model.mInOutDatasets,
+ new InferenceInOutSequence.FromAssets[0], // No inputs for inference.
+ null,
model.mTestName,
model.mModelFile,
null, // Disable evaluation.
@@ -66,10 +71,12 @@
@Test
@LargeTest
public void stressTestNNAPI() throws IOException {
+ Log.i(TAG, mModel.getTestName());
setUseNNApi(true);
- setCompleteInputSet(false);
- TestAction ta = new TestAction(mModel, STRESS_TEST_WARMUP_SECONDS,
- STRESS_TEST_RUNTIME_SECONDS);
- runTest(ta, mModel.getTestName());
+ setCompleteInputSet(true);
+ TestAction ta = new TestAction(mModel, WARMUP_SECONDS, INFERENCE_SECONDS);
+ while (stopwatch.runtime(TimeUnit.SECONDS) < RUNTIME_SECONDS) {
+ runTest(ta, mModel.getTestName());
+ }
}
}
