Merge "Test execution of large number of random models"
diff --git a/README.txt b/README.txt
index 61e7d12..99042b9 100644
--- a/README.txt
+++ b/README.txt
@@ -69,4 +69,6 @@
* memory-mapped-model-load-stress: runs a series of parallel model compilation with memory mapped
TFLite models
-* model-load-random-stress: test compiling a large set of randomly generated models
\ No newline at end of file
+* model-load-random-stress: test compiling a large set of randomly generated models
+
+* inference-random-stress: test running a large set of randomly generated models
\ No newline at end of file
diff --git a/build_and_run_benchmark.sh b/build_and_run_benchmark.sh
index 2b9a401..7cf7f3b 100755
--- a/build_and_run_benchmark.sh
+++ b/build_and_run_benchmark.sh
@@ -48,6 +48,7 @@
MODE="${1:-scoring}"
INSTALL_NATIVE_TESTS=false
+CRASH_TEST_APP="NeuralNetworksApiCrashTest"
APP="NeuralNetworksApiBenchmark"
case "$MODE" in
scoring)
@@ -61,41 +62,45 @@
;;
parallel-inference-stress)
CLASS=com.android.nn.crashtest.app.NNParallelCrashResistantInferenceTest
- APP="NeuralNetworksApiCrashTest"
+ APP="$CRASH_TEST_APP"
;;
parallel-inference-stress-in-process)
CLASS=com.android.nn.crashtest.app.NNParallelInProcessInferenceTest
- APP="NeuralNetworksApiCrashTest"
+ APP="$CRASH_TEST_APP"
;;
client-early-termination-stress)
CLASS=com.android.nn.crashtest.app.NNClientEarlyTerminationTest
- APP="NeuralNetworksApiCrashTest"
+ APP="$CRASH_TEST_APP"
;;
multi-process-inference-stress)
CLASS=com.android.nn.crashtest.app.NNMultipleProcessInferenceTest
- APP="NeuralNetworksApiCrashTest"
+ APP="$CRASH_TEST_APP"
INSTALL_NATIVE_TESTS=true
;;
multi-process-model-load-stress)
CLASS=com.android.nn.crashtest.app.NNMultipleProcessModelLoadTest
- APP="NeuralNetworksApiCrashTest"
+ APP="$CRASH_TEST_APP"
INSTALL_NATIVE_TESTS=true
;;
memory-mapped-model-load-stress)
CLASS=com.android.nn.crashtest.app.NNMemoryMappedModelCompilationTest
- APP="NeuralNetworksApiCrashTest"
+ APP="$CRASH_TEST_APP"
;;
model-load-random-stress)
- APP="NeuralNetworksApiCrashTest"
+ APP="$CRASH_TEST_APP"
CLASS=com.android.nn.crashtest.app.NNRandomGraphLoadTest
;;
+ inference-random-stress)
+ APP="$CRASH_TEST_APP"
+ CLASS=com.android.nn.crashtest.app.NNRandomGraphExecutionTest
+ ;;
*)
echo "Unknown execution mode: $1"
echo "Known modes: scoring (default), inference-stress, model-loading-stress, " \
"parallel-inference-stress, parallel-inference-stress-in-process, " \
"client-early-termination-stress, multi-process-inference-stress, " \
"multi-process-model-load-stress memory-mapped-model-load-stress, " \
- "model-load-random-stress"
+ "model-load-random-stress inference-random-stress"
exit 1
;;
esac
diff --git a/src/com/android/nn/crashtest/app/NNRandomGraphExecutionTest.java b/src/com/android/nn/crashtest/app/NNRandomGraphExecutionTest.java
new file mode 100644
index 0000000..e05cd99
--- /dev/null
+++ b/src/com/android/nn/crashtest/app/NNRandomGraphExecutionTest.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.nn.crashtest.app;
+
+import com.android.nn.crashtest.app.NNRandomGraphTest;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.time.Duration;
+
+@RunWith(Parameterized.class)
+public class NNRandomGraphExecutionTest extends NNRandomGraphTest {
+ public NNRandomGraphExecutionTest(int modelCount, int graphSize, int dimensionRange,
+ Duration duration, String acceleratorName) {
+ super(modelCount, graphSize, dimensionRange, duration,
+ acceleratorName, /*runModelCompilationOnly=*/false);
+ }
+}
\ No newline at end of file
diff --git a/src/com/android/nn/crashtest/app/NNRandomGraphLoadTest.java b/src/com/android/nn/crashtest/app/NNRandomGraphLoadTest.java
index 80de3ce..34962f4 100644
--- a/src/com/android/nn/crashtest/app/NNRandomGraphLoadTest.java
+++ b/src/com/android/nn/crashtest/app/NNRandomGraphLoadTest.java
@@ -16,114 +16,16 @@
package com.android.nn.crashtest.app;
-
-import static com.android.nn.crashtest.app.CrashTestStatus.TestResult.SUCCESS;
-
-import android.content.Intent;
-import android.test.ActivityInstrumentationTestCase2;
-import android.test.UiThreadTest;
-import android.test.suitebuilder.annotation.LargeTest;
-
-import androidx.test.InstrumentationRegistry;
-
-import com.android.nn.benchmark.app.BenchmarkTestBase;
-import com.android.nn.benchmark.core.NnApiDelegationFailure;
-import com.android.nn.benchmark.core.TestModels;
-import com.android.nn.crashtest.core.test.RandomGraphTest;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
import java.time.Duration;
-import java.util.Arrays;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
@RunWith(Parameterized.class)
-public class NNRandomGraphLoadTest
- extends ActivityInstrumentationTestCase2<NNRandomGraphTestActivity>
- implements AcceleratorSpecificTestSupport {
- private static final String TAG = "NN_RAND_MODEL";
-
- protected final String mAcceleratorName;
- private final int mModelCount;
- private final int mGraphSize;
- private final Duration mDuration;
- private final int mDimensionRange;
-
- private final static int SMALL_MODEL_SIZE = 10;
- private final static int LARGE_MODEL_SIZE = 600;
- private final static int NARROW_DIMENSIONS_RANGE = 10;
- private final static int WIDE_DIMENSIONS_RANGE = 1000;
- private final static Duration MAX_TEST_DURATION = Duration.ofMinutes(15);
- private final static int NUMBER_OF_MODELS = 600;
-
- @Parameters(name =
- "{0} models of size {1} and dimensions range {2} for max duration of {3} on accelerator"
- + " {4}")
- public static Iterable<Object[]> testConfiguration() {
- return AcceleratorSpecificTestSupport.perAcceleratorTestConfig(
- Arrays.asList(
- new Object[]{NUMBER_OF_MODELS, SMALL_MODEL_SIZE, WIDE_DIMENSIONS_RANGE,
- MAX_TEST_DURATION},
- new Object[]{NUMBER_OF_MODELS, LARGE_MODEL_SIZE, NARROW_DIMENSIONS_RANGE,
- MAX_TEST_DURATION}));
- }
-
- @Rule
- public TestName mTestName = new TestName();
-
+public class NNRandomGraphLoadTest extends NNRandomGraphTest {
public NNRandomGraphLoadTest(int modelCount, int graphSize, int dimensionRange,
Duration duration, String acceleratorName) {
- super(NNRandomGraphTestActivity.class);
- mModelCount = modelCount;
- mGraphSize = graphSize;
- mDuration = duration;
- mAcceleratorName = acceleratorName;
- mDimensionRange = dimensionRange;
- }
-
- @Before
- @Override
- public void setUp() {
- injectInstrumentation(InstrumentationRegistry.getInstrumentation());
- BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 60);
- setActivityIntent(getLoadModelsOfSizeAndRangeForMaxTimeIntent(mGraphSize, mDimensionRange,
- mModelCount, mAcceleratorName, mDuration, mTestName.getMethodName()));
- }
-
- protected Optional<TestModels.TestModelEntry> findModelForLivenessTest()
- throws NnApiDelegationFailure {
- return findTestModelRunningOnAccelerator(
- getInstrumentation().getTargetContext(), mAcceleratorName);
- }
-
- @Test
- @LargeTest
- @UiThreadTest
- public void testDriverDoesNotFailWithParallelWorkload()
- throws ExecutionException, InterruptedException, NnApiDelegationFailure {
- final NNRandomGraphTestActivity activity = getActivity();
-
- assertEquals(SUCCESS, activity.testResult());
- }
-
- /**
- * @return the intent to use to initialise the RunModelsInMultipleProcesses test class
- */
- protected Intent getLoadModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange,
- int modelsCount, String deviceName, Duration maxTestDuration, String testName) {
- Intent result = new Intent();
- RandomGraphTest
- .intentInitializer(graphSize, dimensionsRange, modelsCount,
- RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS,
- /*compilation only=*/true, deviceName, maxTestDuration.toMillis(), testName)
- .addIntentParams(result);
- return result;
+ super(modelCount, graphSize, dimensionRange, duration,
+ acceleratorName, /*runModelCompilationOnly=*/true);
}
}
\ No newline at end of file
diff --git a/src/com/android/nn/crashtest/app/NNRandomGraphTest.java b/src/com/android/nn/crashtest/app/NNRandomGraphTest.java
new file mode 100644
index 0000000..86e3b0f
--- /dev/null
+++ b/src/com/android/nn/crashtest/app/NNRandomGraphTest.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.nn.crashtest.app;
+
+import static com.android.nn.crashtest.app.CrashTestStatus.TestResult.SUCCESS;
+
+import android.content.Intent;
+import android.test.ActivityInstrumentationTestCase2;
+import android.test.UiThreadTest;
+import android.test.suitebuilder.annotation.LargeTest;
+
+import androidx.test.InstrumentationRegistry;
+
+import com.android.nn.benchmark.app.BenchmarkTestBase;
+import com.android.nn.benchmark.core.NnApiDelegationFailure;
+import com.android.nn.benchmark.core.TestModels;
+import com.android.nn.crashtest.core.test.RandomGraphTest;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+@RunWith(Parameterized.class)
+public abstract class NNRandomGraphTest
+ extends ActivityInstrumentationTestCase2<NNRandomGraphTestActivity>
+ implements AcceleratorSpecificTestSupport {
+ private static final String TAG = "NN_RAND_MODEL";
+
+ protected final String mAcceleratorName;
+ private final int mModelCount;
+ private final int mGraphSize;
+ private final Duration mDuration;
+ private final int mDimensionRange;
+
+ private final static int SMALL_MODEL_SIZE = 10;
+ private final static int LARGE_MODEL_SIZE = 600;
+ private final static int NARROW_DIMENSIONS_RANGE = 10;
+ private final static int WIDE_DIMENSIONS_RANGE = 1000;
+ private final static Duration MAX_TEST_DURATION = Duration.ofMinutes(15);
+ private final static int NUMBER_OF_MODELS = 600;
+ private final boolean mRunModelCompilationOnly;
+
+ @Parameters(name = "{0} models of size {1} and dimensions range {2} for max duration of {3} "
+ + "on accelerator {4}")
+ public static Iterable<Object[]> testConfiguration() {
+ return AcceleratorSpecificTestSupport.perAcceleratorTestConfig(
+ Arrays.asList(
+ new Object[]{NUMBER_OF_MODELS, SMALL_MODEL_SIZE, WIDE_DIMENSIONS_RANGE,
+ MAX_TEST_DURATION},
+ new Object[]{NUMBER_OF_MODELS, LARGE_MODEL_SIZE, NARROW_DIMENSIONS_RANGE,
+ MAX_TEST_DURATION}));
+ }
+
+ @Rule
+ public TestName mTestName = new TestName();
+
+ public NNRandomGraphTest(int modelCount, int graphSize, int dimensionRange,
+ Duration duration, String acceleratorName, boolean runModelCompilationOnly) {
+ super(NNRandomGraphTestActivity.class);
+ mModelCount = modelCount;
+ mGraphSize = graphSize;
+ mDuration = duration;
+ mAcceleratorName = acceleratorName;
+ mDimensionRange = dimensionRange;
+ mRunModelCompilationOnly = runModelCompilationOnly;
+ }
+
+ @Before
+ @Override
+ public void setUp() {
+ injectInstrumentation(InstrumentationRegistry.getInstrumentation());
+ BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 60);
+ setActivityIntent(getTestModelsOfSizeAndRangeForMaxTimeIntent(mGraphSize, mDimensionRange,
+ mModelCount, mAcceleratorName, mDuration, mTestName.getMethodName()));
+ }
+
+ protected Optional<TestModels.TestModelEntry> findModelForLivenessTest()
+ throws NnApiDelegationFailure {
+ return findTestModelRunningOnAccelerator(
+ getInstrumentation().getTargetContext(), mAcceleratorName);
+ }
+
+ @Test
+ @LargeTest
+ @UiThreadTest
+ public void testDriverDoesNotFailWithParallelWorkload()
+ throws ExecutionException, InterruptedException, NnApiDelegationFailure {
+ final NNRandomGraphTestActivity activity = getActivity();
+
+ assertEquals(SUCCESS, activity.testResult());
+ }
+
+ /**
+ * @return the intent to use to initialise the RandomGraphTest test class
+ */
+ protected Intent getTestModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange,
+ int modelsCount, String deviceName, Duration maxTestDuration, String testName) {
+ Intent result = new Intent();
+ RandomGraphTest
+ .intentInitializer(graphSize, dimensionsRange, modelsCount,
+ RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS,
+ mRunModelCompilationOnly, deviceName, maxTestDuration.toMillis(), testName)
+ .addIntentParams(result);
+ return result;
+ }
+}
\ No newline at end of file
