Snap for 6686656 from c192a20f81d085674cf8d69d8c4a560b51b81af3 to sdk-release
Change-Id: Ifcf1c503aa5543c54fd35ffb9ac45c5e46d342fa
diff --git a/results/generate_result.py b/results/generate_result.py
index 1488c20..7358a51 100755
--- a/results/generate_result.py
+++ b/results/generate_result.py
@@ -28,6 +28,7 @@
import csv
import os
import re
+import math
class ScoreException(Exception):
@@ -54,6 +55,8 @@
(re.compile('mobilenet_v1.*'), 'MobileNet v1 Float'),
(re.compile('mobilenet_v2.*quant.*'), 'MobileNet v2 Quantized'),
(re.compile('mobilenet_v2.*'), 'MobileNet v2 Float'),
+ (re.compile('mobilenet_v3.*uint8.*'), 'MobileNet v3 Quantized'),
+ (re.compile('mobilenet_v3.*'), 'MobileNet v3 Float'),
(re.compile('tts.*'), 'LSTM Text-to-speech'),
(re.compile('asr.*'), 'LSTM Automatic Speech Recognition'),
]
@@ -146,9 +149,13 @@
def get_frequency_graph(time_freq_start_sec, time_freq_step_sec, time_freq_sec,
start_sec, end_sec):
"""Generate input x/y data for latency frequency graph."""
- left_to_pad = int((time_freq_start_sec - start_sec) / time_freq_step_sec)
+ left_to_pad = (int((time_freq_start_sec - start_sec) / time_freq_step_sec)
+ if time_freq_step_sec != 0
+ else math.inf)
end_time = time_freq_start_sec + len(time_freq_sec) * time_freq_step_sec
- right_to_pad = int((end_sec - end_time) / time_freq_step_sec)
+ right_to_pad = (int((end_sec - end_time) / time_freq_step_sec)
+ if time_freq_step_sec != 0
+ else math.inf)
# After pading more that 64 values, graphs start to look messy,
# bail out in that case.
diff --git a/src/com/android/nn/crashtest/app/NNCrashTestActivity.java b/src/com/android/nn/crashtest/app/NNCrashTestActivity.java
new file mode 100644
index 0000000..b1ae0fe
--- /dev/null
+++ b/src/com/android/nn/crashtest/app/NNCrashTestActivity.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.nn.crashtest.app;
+
+import android.app.Activity;
+import android.content.Intent;
+import android.util.Log;
+
+import com.android.nn.crashtest.core.CrashTest;
+import com.android.nn.crashtest.core.CrashTestCoordinator;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+public abstract class NNCrashTestActivity extends Activity {
+ public static final Duration MAX_TEST_DELAY_BEFORE_HANG = Duration.ofSeconds(30);
+
+ private final CrashTestStatus mTestStatus = new CrashTestStatus(this::logMessage);
+ private final CrashTestCoordinator mCoordinator = new CrashTestCoordinator(this);
+ private Duration mDuration;
+
+ protected void logMessage(String msg) {
+ Log.i(getTag(), msg);
+ }
+
+ protected abstract String getTag();
+
+ protected abstract String getTestName(Intent intent);
+
+ protected abstract long getTestDurationMillis(Intent intent);
+
+ protected abstract CrashTestCoordinator.CrashTestIntentInitializer getIntentInitializer(Intent intent);
+
+ protected abstract Class<? extends CrashTest> getTestClass();
+
+
+ @Override
+ protected void onResume() {
+ super.onResume();
+
+ final Intent intent = getIntent();
+
+ mDuration = Duration.ofMillis(getTestDurationMillis(intent));
+ mCoordinator.startTest(getTestClass(),
+ getIntentInitializer(intent),
+ mTestStatus,
+ /*separateProcess=*/true, getTestName(intent));
+ }
+
+ // This method blocks until the tests complete and returns true if all tests completed
+ // successfully
+ public CrashTestStatus.TestResult testResult() {
+ try {
+ final Duration testTimeout = mDuration.plus(MAX_TEST_DELAY_BEFORE_HANG);
+ boolean completed =
+ mTestStatus.waitForCompletion(testTimeout.toMillis(), TimeUnit.MILLISECONDS);
+ if (!completed) {
+ Log.w(getTag(), String.format("Test didn't complete within %s. Returning HANG", testTimeout));
+ return CrashTestStatus.TestResult.HANG;
+ }
+ return mTestStatus.result();
+ } catch (InterruptedException e) {
+ Log.w(getTag(), "Interrupted while waiting for test completion. Returning HANG");
+ return CrashTestStatus.TestResult.HANG;
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/com/android/nn/crashtest/app/NNMultiProcessTestActivity.java b/src/com/android/nn/crashtest/app/NNMultiProcessTestActivity.java
index fb34ad4..026d070 100644
--- a/src/com/android/nn/crashtest/app/NNMultiProcessTestActivity.java
+++ b/src/com/android/nn/crashtest/app/NNMultiProcessTestActivity.java
@@ -19,59 +19,39 @@
import android.app.Activity;
import android.content.Intent;
import android.util.Log;
+
+import com.android.nn.crashtest.core.CrashTest;
import com.android.nn.crashtest.core.CrashTestCoordinator;
import com.android.nn.crashtest.core.test.RunModelsInMultipleProcesses;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
-public class NNMultiProcessTestActivity extends Activity {
+public class NNMultiProcessTestActivity extends NNCrashTestActivity {
private static final String TAG = "NNMultiProcessTest";
- public static final Duration MAX_TEST_DELAY_BEFORE_HANG = Duration.ofSeconds(30);
- private final CrashTestStatus mTestStatus = new CrashTestStatus(this::logMessage);
- private final CrashTestCoordinator mCoordinator = new CrashTestCoordinator(this);
- private Duration mDuration;
-
- protected void logMessage(String msg) {
- Log.i(TAG, msg);
+ @Override
+ protected String getTag() {
+ return TAG;
}
@Override
- protected void onResume() {
- super.onResume();
-
- final Intent intent = getIntent();
-
- mDuration = Duration.ofMillis(intent.getLongExtra(
- RunModelsInMultipleProcesses.TEST_DURATION, Duration.ofSeconds(30).toMillis()));
- mCoordinator.startTest(RunModelsInMultipleProcesses.class,
- RunModelsInMultipleProcesses.intentInitializer(
- intent.getStringExtra(RunModelsInMultipleProcesses.TEST_NAME),
- intent.getStringExtra(RunModelsInMultipleProcesses.MODEL_NAME),
- intent.getIntExtra(RunModelsInMultipleProcesses.PROCESSES, 3),
- intent.getIntExtra(RunModelsInMultipleProcesses.THREADS, 1), mDuration,
- intent.getStringExtra(RunModelsInMultipleProcesses.NNAPI_DEVICE_NAME),
- intent.getBooleanExtra(RunModelsInMultipleProcesses.JUST_COMPILE, false),
- intent.getIntExtra(RunModelsInMultipleProcesses.CLIENT_FAILURE_RATE_PERCENT, 0)),
- mTestStatus,
- /*separateProcess=*/false, intent.getStringExtra(RunModelsInMultipleProcesses.TEST_NAME));
+ protected String getTestName(Intent intent) {
+ return intent.getStringExtra(RunModelsInMultipleProcesses.TEST_NAME);
}
- // This method blocks until the tests complete and returns true if all tests completed
- // successfully
- public CrashTestStatus.TestResult testResult() {
- try {
- final Duration testTimeout = mDuration.plus(MAX_TEST_DELAY_BEFORE_HANG);
- boolean completed =
- mTestStatus.waitForCompletion(testTimeout.toMillis(), TimeUnit.MILLISECONDS);
- if (!completed) {
- Log.w(TAG, String.format("Test didn't comoplete within %s. Returning HANG", testTimeout));
- return CrashTestStatus.TestResult.HANG;
- }
- return mTestStatus.result();
- } catch (InterruptedException e) {
- Log.w(TAG, "Interrupted while waiting for test completion. Returning HANG");
- return CrashTestStatus.TestResult.HANG;
- }
+ @Override
+ protected long getTestDurationMillis(Intent intent) {
+ return intent.getLongExtra(
+ RunModelsInMultipleProcesses.TEST_DURATION, RunModelsInMultipleProcesses.DEFAULT_TEST_DURATION);
+ }
+
+ @Override
+ protected CrashTestCoordinator.CrashTestIntentInitializer getIntentInitializer(Intent intent) {
+ return RunModelsInMultipleProcesses.intentInitializer(intent);
+ }
+
+ @Override
+ protected Class<? extends CrashTest> getTestClass() {
+ return RunModelsInMultipleProcesses.class;
}
}
\ No newline at end of file
diff --git a/src/com/android/nn/crashtest/app/NNRandomGraphTestActivity.java b/src/com/android/nn/crashtest/app/NNRandomGraphTestActivity.java
index 76e2f64..d533532 100644
--- a/src/com/android/nn/crashtest/app/NNRandomGraphTestActivity.java
+++ b/src/com/android/nn/crashtest/app/NNRandomGraphTestActivity.java
@@ -16,68 +16,39 @@
package com.android.nn.crashtest.app;
-import android.app.Activity;
import android.content.Intent;
-import android.util.Log;
-
+import com.android.nn.crashtest.core.CrashTest;
import com.android.nn.crashtest.core.CrashTestCoordinator;
import com.android.nn.crashtest.core.test.RandomGraphTest;
-import java.time.Duration;
-import java.util.concurrent.TimeUnit;
-
-public class NNRandomGraphTestActivity extends Activity {
+public class NNRandomGraphTestActivity extends NNCrashTestActivity {
private static final String TAG = "NN_RAND_MODEL";
- private final CrashTestStatus mTestStatus = new CrashTestStatus(this::logMessage);
- private final CrashTestCoordinator mCoordinator = new CrashTestCoordinator(this);
- private Duration mDuration;
- protected void logMessage(String msg) {
- Log.i(TAG, msg);
+ @Override
+ protected String getTag() {
+ return TAG;
}
@Override
- protected void onResume() {
- super.onResume();
-
- final Intent intent = getIntent();
-
- mDuration = Duration.ofMillis(intent.getLongExtra(
- RandomGraphTest.MAX_TEST_DURATION, RandomGraphTest.DEFAULT_MAX_TEST_DURATION_MILLIS));
- mCoordinator.startTest(RandomGraphTest.class,
- RandomGraphTest.intentInitializer(
- intent.getIntExtra(RandomGraphTest.GRAPH_SIZE, RandomGraphTest.DEFAULT_GRAPH_SIZE),
- intent.getIntExtra(
- RandomGraphTest.DIMENSIONS_RANGE, RandomGraphTest.DEFAULT_DIMENSIONS_RANGE),
- intent.getIntExtra(RandomGraphTest.MODELS_COUNT, RandomGraphTest.DEFAULT_MODELS_COUNT),
- intent.getLongExtra(RandomGraphTest.PAUSE_BETWEEN_MODELS_MS,
- RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS),
- intent.getBooleanExtra(
- RandomGraphTest.COMPILATION_ONLY, RandomGraphTest.DEFAULT_COMPILATION_ONLY),
- intent.getStringExtra(RandomGraphTest.DEVICE_NAME),
- mDuration.toMillis(),
- intent.getStringExtra(RandomGraphTest.TEST_NAME)),
- mTestStatus,
- /*separateProcess=*/true, intent.getStringExtra(RandomGraphTest.TEST_NAME));
+ protected String getTestName(Intent intent) {
+ return intent.getStringExtra(RandomGraphTest.TEST_NAME);
}
- // This method blocks until the tests complete and returns true if all tests completed
- // successfully
- public CrashTestStatus.TestResult testResult() {
- try {
- final Duration testTimeout = mDuration.plus(Duration.ofSeconds(15));
- boolean completed =
- mTestStatus.waitForCompletion(testTimeout.toMillis(), TimeUnit.MILLISECONDS);
- if (!completed) {
- Log.w(TAG, String.format("Test didn't comoplete within %s. Returning HANG", testTimeout));
- return CrashTestStatus.TestResult.HANG;
- }
- return mTestStatus.result();
- } catch (InterruptedException e) {
- Log.w(TAG, "Interrupted while waiting for test completion. Returning HANG");
- return CrashTestStatus.TestResult.HANG;
- }
+ @Override
+ protected long getTestDurationMillis(Intent intent) {
+ return intent.getLongExtra(
+ RandomGraphTest.MAX_TEST_DURATION, RandomGraphTest.DEFAULT_MAX_TEST_DURATION_MILLIS);
+ }
+
+ @Override
+ protected CrashTestCoordinator.CrashTestIntentInitializer getIntentInitializer(Intent intent) {
+ return RandomGraphTest.intentInitializer(intent);
+ }
+
+ @Override
+ protected Class<? extends CrashTest> getTestClass() {
+ return RandomGraphTest.class;
}
}
\ No newline at end of file
diff --git a/src/com/android/nn/crashtest/core/test/RandomGraphTest.java b/src/com/android/nn/crashtest/core/test/RandomGraphTest.java
index c5403ca..7abfb53 100644
--- a/src/com/android/nn/crashtest/core/test/RandomGraphTest.java
+++ b/src/com/android/nn/crashtest/core/test/RandomGraphTest.java
@@ -123,6 +123,24 @@
};
}
+ static public CrashTestCoordinator.CrashTestIntentInitializer intentInitializer(
+ Intent copyFrom) {
+ return intentInitializer(
+ copyFrom.getIntExtra(RandomGraphTest.GRAPH_SIZE,
+ RandomGraphTest.DEFAULT_GRAPH_SIZE),
+ copyFrom.getIntExtra(
+ RandomGraphTest.DIMENSIONS_RANGE, RandomGraphTest.DEFAULT_DIMENSIONS_RANGE),
+ copyFrom.getIntExtra(RandomGraphTest.MODELS_COUNT,
+ RandomGraphTest.DEFAULT_MODELS_COUNT),
+ copyFrom.getLongExtra(RandomGraphTest.PAUSE_BETWEEN_MODELS_MS,
+ RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS),
+ copyFrom.getBooleanExtra(
+ RandomGraphTest.COMPILATION_ONLY, RandomGraphTest.DEFAULT_COMPILATION_ONLY),
+ copyFrom.getStringExtra(RandomGraphTest.DEVICE_NAME),
+ copyFrom.getLongExtra(MAX_TEST_DURATION,
+ DEFAULT_MAX_TEST_DURATION_MILLIS),
+ copyFrom.getStringExtra(RandomGraphTest.TEST_NAME));
+ }
private Context mContext;
private String mDeviceName;
diff --git a/src/com/android/nn/crashtest/core/test/RunModelsInMultipleProcesses.java b/src/com/android/nn/crashtest/core/test/RunModelsInMultipleProcesses.java
index 320c31c..d23564e 100644
--- a/src/com/android/nn/crashtest/core/test/RunModelsInMultipleProcesses.java
+++ b/src/com/android/nn/crashtest/core/test/RunModelsInMultipleProcesses.java
@@ -47,6 +47,11 @@
public static final String NNAPI_DEVICE_NAME = "nnapi_device_name";
public static final String JUST_COMPILE = "just_compile";
public static final String CLIENT_FAILURE_RATE_PERCENT = "client_failure_rate_percent";
+ public static final long DEFAULT_TEST_DURATION = Duration.ofSeconds(60).toMillis();
+ public static final int DEFAULT_PROCESSES = 3;
+ public static final int DEFAULT_THREADS = 1;
+ public static final boolean DEFAULT_JUST_COMPILE = false;
+ public static final int DEFAULT_CLIENT_FAILURE_RATE_PERCENT = 0;
private Context mContext;
private int mThreadCount;
@@ -72,6 +77,20 @@
intent.putExtra(CLIENT_FAILURE_RATE_PERCENT, clientFailureRatePercent);
};
}
+ static public CrashTestCoordinator.CrashTestIntentInitializer intentInitializer(
+ Intent copyFrom) {
+ return intentInitializer(
+ copyFrom.getStringExtra(RunModelsInMultipleProcesses.TEST_NAME),
+ copyFrom.getStringExtra(RunModelsInMultipleProcesses.MODEL_NAME),
+ copyFrom.getIntExtra(RunModelsInMultipleProcesses.PROCESSES, DEFAULT_PROCESSES),
+ copyFrom.getIntExtra(RunModelsInMultipleProcesses.THREADS, DEFAULT_THREADS),
+ Duration.ofMillis(copyFrom.getLongExtra(TEST_DURATION, DEFAULT_TEST_DURATION)),
+ copyFrom.getStringExtra(RunModelsInMultipleProcesses.NNAPI_DEVICE_NAME),
+ copyFrom.getBooleanExtra(RunModelsInMultipleProcesses.JUST_COMPILE,
+ DEFAULT_JUST_COMPILE),
+ copyFrom.getIntExtra(RunModelsInMultipleProcesses.CLIENT_FAILURE_RATE_PERCENT,
+ DEFAULT_CLIENT_FAILURE_RATE_PERCENT));
+ }
@Override
public void init(
@@ -79,13 +98,14 @@
mContext = context;
mTestName = configParams.getStringExtra(TEST_NAME);
mTestModelEntry = TestModels.getModelByName(configParams.getStringExtra(MODEL_NAME));
- mProcessCount = configParams.getIntExtra(PROCESSES, 3);
- mThreadCount = configParams.getIntExtra(THREADS, 1);
+ mProcessCount = configParams.getIntExtra(PROCESSES, DEFAULT_PROCESSES);
+ mThreadCount = configParams.getIntExtra(THREADS, DEFAULT_THREADS);
mTestDuration = Duration.ofMillis(
- configParams.getLongExtra(TEST_DURATION, Duration.ofSeconds(60).toMillis()));
+ configParams.getLongExtra(TEST_DURATION, DEFAULT_TEST_DURATION));
mNnApiDeviceName = configParams.getStringExtra(NNAPI_DEVICE_NAME);
- mJustCompileModel = configParams.getBooleanExtra(JUST_COMPILE, false);
- mClientFailureRatePercent = configParams.getIntExtra(CLIENT_FAILURE_RATE_PERCENT, 0);
+ mJustCompileModel = configParams.getBooleanExtra(JUST_COMPILE, DEFAULT_JUST_COMPILE);
+ mClientFailureRatePercent = configParams.getIntExtra(CLIENT_FAILURE_RATE_PERCENT,
+ DEFAULT_CLIENT_FAILURE_RATE_PERCENT);
}
private void deleteOrWarn(File fileToDelete) {
