Merge "CEC VTS: Add testSendRandomMessage"
diff --git a/neuralnetworks/V1_2/benchmark/Android.bp b/neuralnetworks/V1_2/benchmark/Android.bp
deleted file mode 100644
index ad22ebd..0000000
--- a/neuralnetworks/V1_2/benchmark/Android.bp
+++ /dev/null
@@ -1,19 +0,0 @@
-//
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-vts_config {
- name: "VtsHalNeuralnetworksV1_2Benchmark",
-}
diff --git a/neuralnetworks/V1_2/benchmark/AndroidTest.xml b/neuralnetworks/V1_2/benchmark/AndroidTest.xml
deleted file mode 100644
index b768594..0000000
--- a/neuralnetworks/V1_2/benchmark/AndroidTest.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Copyright (C) 2019 The Android Open Source Project
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration description="Config for VTS VtsHalNeuralnetworksV1_2Benchmark test cases">
- <option name="config-descriptor:metadata" key="plan" value="vts-hal" />
- <target_preparer class="com.android.compatibility.common.tradefed.targetprep.VtsFilePusher">
- <option name="push-group" value="HostDrivenTest.push" />
- <option name="push-group" value="HalHidlTargetTest.push"/>
- </target_preparer>
- <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
- <option name="cleanup-apks" value="true" />
- <option name="test-file-name" value="VtsHalNeuralnetworksV1_2BenchmarkTestCases.apk" />
- </target_preparer>
- <test class="com.android.tradefed.testtype.VtsMultiDeviceTest">
- <option name="test-module-name" value="VtsHalNeuralnetworksV1_2Benchmark"/>
- <option name="test-case-path" value="vts/testcases/hal/neuralnetworks/V1_2/benchmark/VtsHalNeuralnetworksV1_2Benchmark" />
- <option name="test-timeout" value="1h"/>
- </test>
-</configuration>
diff --git a/neuralnetworks/V1_2/benchmark/VtsHalNeuralnetworksV1_2Benchmark.py b/neuralnetworks/V1_2/benchmark/VtsHalNeuralnetworksV1_2Benchmark.py
deleted file mode 100644
index d0e4b3a..0000000
--- a/neuralnetworks/V1_2/benchmark/VtsHalNeuralnetworksV1_2Benchmark.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2019 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import logging
-import re
-
-from time import sleep
-from vts.runners.host import asserts
-from vts.runners.host import const
-from vts.runners.host import test_runner
-from vts.testcases.template.hal_hidl_host_test import hal_hidl_host_test
-
-
-class VtsHalNeuralnetworksV1_2Benchmark(hal_hidl_host_test.HalHidlHostTest):
- """A test case that runs the accuracy benchmark for every NN device driver."""
-
- TEST_HAL_SERVICES = {"android.hardware.neuralnetworks@1.2::IDevice"}
-
- def testAccuracy(self):
- """Test that the driver meets accuracy requirements."""
- hal_service_instance = self.cur_param[0]
- command = (
- "am instrument -e halServiceInstance '{}' " +
- "-w com.android.nn.benchmark.vts.v1_2/androidx.test.runner.AndroidJUnitRunner"
- ).format(hal_service_instance)
- logging.info("Executing command: %s", command)
- logging.info("Check logcat for more information")
- results = self.shell.Execute([command])
- stdout = results[const.STDOUT][0]
- stderr = results[const.STDERR][0]
- exit_code = results[const.EXIT_CODE][0]
- if stdout:
- logging.info("stdout: %s", stdout)
- if stderr:
- logging.error("stderr: %s", stderr)
- if exit_code:
- logging.error("exit_code: %s", exit_code)
- # The "am instrument" command exits with 0 even when there are failed tests.
- asserts.assertTrue(re.search(r"^OK \([0-9]+ tests?\)$", stdout, re.M),
- "Benchmark failed")
-
-
-if __name__ == "__main__":
- test_runner.main()
diff --git a/neuralnetworks/V1_2/benchmark/__init__.py b/neuralnetworks/V1_2/benchmark/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/neuralnetworks/V1_2/benchmark/__init__.py
+++ /dev/null
diff --git a/neuralnetworks/V1_2/benchmark/java/Android.mk b/neuralnetworks/V1_2/benchmark/java/Android.mk
index 53d2cbc..4407809 100644
--- a/neuralnetworks/V1_2/benchmark/java/Android.mk
+++ b/neuralnetworks/V1_2/benchmark/java/Android.mk
@@ -26,17 +26,16 @@
# Include both the 32 and 64 bit versions
LOCAL_MULTILIB := both
-# TODO: This is from the CTS app. Figure out the proper way to do this in VTS.
# Tag this module as a cts test artifact
-LOCAL_COMPATIBILITY_SUITE := cts vts10
+LOCAL_COMPATIBILITY_SUITE := vts vts10
-LOCAL_STATIC_JAVA_LIBRARIES := androidx.test.rules \
+LOCAL_STATIC_JAVA_LIBRARIES := androidx.test.rules android.hidl.manager-V1.2-java \
compatibility-device-util-axt ctstestrunner-axt junit NeuralNetworksApiBenchmark_Lib
LOCAL_JNI_SHARED_LIBRARIES := libnnbenchmark_jni
LOCAL_SRC_FILES := $(call all-java-files-under, src)
LOCAL_ASSET_DIR := test/mlts/models/assets
-LOCAL_SDK_VERSION := current
+LOCAL_SDK_VERSION := system_current
include $(BUILD_CTS_PACKAGE)
diff --git a/neuralnetworks/V1_2/benchmark/java/AndroidTest.xml b/neuralnetworks/V1_2/benchmark/java/AndroidTest.xml
new file mode 100644
index 0000000..4c258c8
--- /dev/null
+++ b/neuralnetworks/V1_2/benchmark/java/AndroidTest.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2020 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Config for VtsHalNeuralnetworksV1_2BenchmarkTestCases">
+ <option name="test-suite-tag" value="cts" />
+ <option name="config-descriptor:metadata" key="component" value="neuralnetworks" />
+ <option name="config-descriptor:metadata" key="parameter" value="not_instant_app" />
+ <option name="config-descriptor:metadata" key="parameter" value="not_multi_abi" />
+ <option name="config-descriptor:metadata" key="parameter" value="not_secondary_user" />
+ <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
+ <option name="cleanup-apks" value="true" />
+ <option name="test-file-name" value="VtsHalNeuralnetworksV1_2BenchmarkTestCases.apk" />
+ </target_preparer>
+ <test class="com.android.tradefed.testtype.AndroidJUnitTest" >
+ <option name="runner" value="androidx.test.runner.AndroidJUnitRunner" />
+ <option name="package" value="com.android.nn.benchmark.vts.v1_2" />
+ </test>
+</configuration>
\ No newline at end of file
diff --git a/neuralnetworks/V1_2/benchmark/java/src/com/android/nn/benchmark/vts/v1_2/NNAccuracyTest.java b/neuralnetworks/V1_2/benchmark/java/src/com/android/nn/benchmark/vts/v1_2/NNAccuracyTest.java
index 723acda..17ca0ba 100644
--- a/neuralnetworks/V1_2/benchmark/java/src/com/android/nn/benchmark/vts/v1_2/NNAccuracyTest.java
+++ b/neuralnetworks/V1_2/benchmark/java/src/com/android/nn/benchmark/vts/v1_2/NNAccuracyTest.java
@@ -20,9 +20,11 @@
import static junit.framework.TestCase.assertTrue;
import android.app.Activity;
+import android.hidl.manager.V1_2.IServiceManager;
+import android.os.RemoteException;
import android.util.Pair;
-import androidx.test.filters.LargeTest;
import androidx.test.InstrumentationRegistry;
+import androidx.test.filters.LargeTest;
import androidx.test.rule.ActivityTestRule;
import com.android.nn.benchmark.core.BenchmarkException;
import com.android.nn.benchmark.core.BenchmarkResult;
@@ -37,31 +39,40 @@
import org.junit.AssumptionViolatedException;
import org.junit.Before;
import org.junit.Rule;
+import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import org.junit.Test;
/**
* Tests the accuracy of the model outputs.
*/
@RunWith(Parameterized.class)
public class NNAccuracyTest {
- private static final String ARG_HAL_SERVICE_INSTANCE = "halServiceInstance";
- private static final String HAL_SERVICE_INSTANCE_PREFIX =
- "android.hardware.neuralnetworks@1.2::IDevice/";
+ private static final String HAL_SERVICE_TYPE = "android.hardware.neuralnetworks@1.2::IDevice";
@Rule
public ActivityTestRule<NNAccuracyActivity> mActivityRule =
new ActivityTestRule<>(NNAccuracyActivity.class);
- @Parameterized.Parameter(0) public TestModels.TestModelEntry mModel;
+ private static class InstanceModel {
+ final String mInstance;
+ final TestModels.TestModelEntry mEntry;
+
+ InstanceModel(String instance, TestModels.TestModelEntry entry) {
+ mInstance = instance;
+ mEntry = entry;
+ }
+
+ @Override
+ public String toString() {
+ return mInstance + ":" + mEntry.mModelName;
+ }
+ }
+ @Parameterized.Parameter(0) public InstanceModel mModel;
private Activity mActivity;
- private String mHalServiceInstance;
- private String mDeviceName;
-
// TODO(vddang): Add mobilenet_v1_0.25_128_quant_topk_aosp
private static final String[] MODEL_NAMES = new String[] {
"tts_float",
@@ -81,37 +92,42 @@
};
@Parameters(name = "{0}")
- public static List<TestModels.TestModelEntry> modelsList() {
- List<TestModels.TestModelEntry> models = new ArrayList<>();
- for (String modelName : MODEL_NAMES) {
- models.add(TestModels.getModelByName(modelName));
+ public static Iterable<InstanceModel> parameterList() throws RemoteException {
+ List<String> serviceInstances =
+ IServiceManager.getService().listManifestByInterface(HAL_SERVICE_TYPE);
+ List<InstanceModel> parameters = new ArrayList<>();
+ for (String instance : serviceInstances) {
+ for (String modelName : MODEL_NAMES) {
+ TestModels.TestModelEntry entry = TestModels.getModelByName(modelName);
+ if (entry == null) {
+ throw new RuntimeException("Failed to find model of name " + modelName);
+ }
+ parameters.add(new InstanceModel(instance, entry));
+ }
}
- return Collections.unmodifiableList(models);
+
+ return Collections.unmodifiableList(parameters);
}
@Before
public void setUp() throws Exception {
mActivity = mActivityRule.getActivity();
- mHalServiceInstance =
- InstrumentationRegistry.getArguments().getString(ARG_HAL_SERVICE_INSTANCE);
- assertTrue(mHalServiceInstance.startsWith(HAL_SERVICE_INSTANCE_PREFIX));
- mDeviceName = mHalServiceInstance.substring(HAL_SERVICE_INSTANCE_PREFIX.length());
}
@Test
@LargeTest
public void testDriver() throws BenchmarkException, IOException {
- NNTestBase test = mModel.createNNTestBase();
+ NNTestBase test = mModel.mEntry.createNNTestBase();
test.useNNApi();
- test.setNNApiDeviceName(mDeviceName);
+ test.setNNApiDeviceName(mModel.mInstance);
if (!test.setupModel(mActivity)) {
throw new AssumptionViolatedException("The driver rejected the model.");
}
Pair<List<InferenceInOutSequence>, List<InferenceResult>> inferenceResults =
test.runBenchmarkCompleteInputSet(/*setRepeat=*/1, /*timeoutSec=*/3600);
- BenchmarkResult benchmarkResult = BenchmarkResult.fromInferenceResults(mModel.mModelName,
- BenchmarkResult.BACKEND_TFLITE_NNAPI, inferenceResults.first,
- inferenceResults.second, test.getEvaluator());
+ BenchmarkResult benchmarkResult = BenchmarkResult.fromInferenceResults(
+ mModel.mEntry.mModelName, BenchmarkResult.BACKEND_TFLITE_NNAPI,
+ inferenceResults.first, inferenceResults.second, test.getEvaluator());
assertFalse(benchmarkResult.hasValidationErrors());
}
}
diff --git a/neuralnetworks/V1_3/benchmark/Android.bp b/neuralnetworks/V1_3/benchmark/Android.bp
deleted file mode 100644
index 8641ba4..0000000
--- a/neuralnetworks/V1_3/benchmark/Android.bp
+++ /dev/null
@@ -1,19 +0,0 @@
-//
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-vts_config {
- name: "VtsHalNeuralnetworksV1_3Benchmark",
-}
diff --git a/neuralnetworks/V1_3/benchmark/AndroidTest.xml b/neuralnetworks/V1_3/benchmark/AndroidTest.xml
deleted file mode 100644
index 9ae16b4..0000000
--- a/neuralnetworks/V1_3/benchmark/AndroidTest.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Copyright (C) 2019 The Android Open Source Project
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration description="Config for VTS VtsHalNeuralnetworksV1_3Benchmark test cases">
- <option name="config-descriptor:metadata" key="plan" value="vts-hal" />
- <target_preparer class="com.android.compatibility.common.tradefed.targetprep.VtsFilePusher">
- <option name="push-group" value="HostDrivenTest.push" />
- <option name="push-group" value="HalHidlTargetTest.push"/>
- </target_preparer>
- <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
- <option name="cleanup-apks" value="true" />
- <option name="test-file-name" value="VtsHalNeuralnetworksV1_3BenchmarkTestCases.apk" />
- </target_preparer>
- <test class="com.android.tradefed.testtype.VtsMultiDeviceTest">
- <option name="test-module-name" value="VtsHalNeuralnetworksV1_3Benchmark"/>
- <option name="test-case-path" value="vts/testcases/hal/neuralnetworks/V1_3/benchmark/VtsHalNeuralnetworksV1_3Benchmark" />
- <option name="test-timeout" value="1h"/>
- </test>
-</configuration>
diff --git a/neuralnetworks/V1_3/benchmark/VtsHalNeuralnetworksV1_3Benchmark.py b/neuralnetworks/V1_3/benchmark/VtsHalNeuralnetworksV1_3Benchmark.py
deleted file mode 100644
index f80d4f2..0000000
--- a/neuralnetworks/V1_3/benchmark/VtsHalNeuralnetworksV1_3Benchmark.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2019 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import logging
-import re
-
-from time import sleep
-from vts.runners.host import asserts
-from vts.runners.host import const
-from vts.runners.host import test_runner
-from vts.testcases.template.hal_hidl_host_test import hal_hidl_host_test
-
-
-class VtsHalNeuralnetworksV1_3Benchmark(hal_hidl_host_test.HalHidlHostTest):
- """A test case that runs the accuracy benchmark for every NN device driver."""
-
- TEST_HAL_SERVICES = {"android.hardware.neuralnetworks@1.3::IDevice"}
-
- def testAccuracy(self):
- """Test that the driver meets accuracy requirements."""
- hal_service_instance = self.cur_param[0]
- command = (
- "am instrument -e halServiceInstance '{}' " +
- "-w com.android.nn.benchmark.vts.v1_3/androidx.test.runner.AndroidJUnitRunner"
- ).format(hal_service_instance)
- logging.info("Executing command: %s", command)
- logging.info("Check logcat for more information")
- results = self.shell.Execute([command])
- stdout = results[const.STDOUT][0]
- stderr = results[const.STDERR][0]
- exit_code = results[const.EXIT_CODE][0]
- if stdout:
- logging.info("stdout: %s", stdout)
- if stderr:
- logging.error("stderr: %s", stderr)
- if exit_code:
- logging.error("exit_code: %s", exit_code)
- # The "am instrument" command exits with 0 even when there are failed tests.
- asserts.assertTrue(re.search(r"^OK \([0-9]+ tests?\)$", stdout, re.M),
- "Benchmark failed")
-
-
-if __name__ == "__main__":
- test_runner.main()
diff --git a/neuralnetworks/V1_3/benchmark/__init__.py b/neuralnetworks/V1_3/benchmark/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/neuralnetworks/V1_3/benchmark/__init__.py
+++ /dev/null
diff --git a/neuralnetworks/V1_3/benchmark/java/Android.mk b/neuralnetworks/V1_3/benchmark/java/Android.mk
index e3bca84..a5beae9 100644
--- a/neuralnetworks/V1_3/benchmark/java/Android.mk
+++ b/neuralnetworks/V1_3/benchmark/java/Android.mk
@@ -26,17 +26,16 @@
# Include both the 32 and 64 bit versions
LOCAL_MULTILIB := both
-# TODO: This is from the CTS app. Figure out the proper way to do this in VTS.
# Tag this module as a cts test artifact
-LOCAL_COMPATIBILITY_SUITE := cts vts10
+LOCAL_COMPATIBILITY_SUITE := vts vts10
-LOCAL_STATIC_JAVA_LIBRARIES := androidx.test.rules \
+LOCAL_STATIC_JAVA_LIBRARIES := androidx.test.rules android.hidl.manager-V1.2-java \
compatibility-device-util-axt ctstestrunner-axt junit NeuralNetworksApiBenchmark_Lib
LOCAL_JNI_SHARED_LIBRARIES := libnnbenchmark_jni
LOCAL_SRC_FILES := $(call all-java-files-under, src)
LOCAL_ASSET_DIR := test/mlts/models/assets
-LOCAL_SDK_VERSION := current
+LOCAL_SDK_VERSION := system_current
include $(BUILD_CTS_PACKAGE)
diff --git a/neuralnetworks/V1_3/benchmark/java/AndroidTest.xml b/neuralnetworks/V1_3/benchmark/java/AndroidTest.xml
new file mode 100644
index 0000000..1fe09c2
--- /dev/null
+++ b/neuralnetworks/V1_3/benchmark/java/AndroidTest.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2020 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Config for VtsHalNeuralnetworksV1_3BenchmarkTestCases">
+ <option name="test-suite-tag" value="cts" />
+ <option name="config-descriptor:metadata" key="component" value="neuralnetworks" />
+ <option name="config-descriptor:metadata" key="parameter" value="not_instant_app" />
+ <option name="config-descriptor:metadata" key="parameter" value="not_multi_abi" />
+ <option name="config-descriptor:metadata" key="parameter" value="not_secondary_user" />
+ <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
+ <option name="cleanup-apks" value="true" />
+ <option name="test-file-name" value="VtsHalNeuralnetworksV1_3BenchmarkTestCases.apk" />
+ </target_preparer>
+ <test class="com.android.tradefed.testtype.AndroidJUnitTest" >
+ <option name="runner" value="androidx.test.runner.AndroidJUnitRunner" />
+ <option name="package" value="com.android.nn.benchmark.vts.v1_3" />
+ </test>
+</configuration>
\ No newline at end of file
diff --git a/neuralnetworks/V1_3/benchmark/java/src/com/android/nn/benchmark/vts/v1_3/NNAccuracyTest.java b/neuralnetworks/V1_3/benchmark/java/src/com/android/nn/benchmark/vts/v1_3/NNAccuracyTest.java
index 561d0fd..bdc4311 100644
--- a/neuralnetworks/V1_3/benchmark/java/src/com/android/nn/benchmark/vts/v1_3/NNAccuracyTest.java
+++ b/neuralnetworks/V1_3/benchmark/java/src/com/android/nn/benchmark/vts/v1_3/NNAccuracyTest.java
@@ -20,6 +20,8 @@
import static junit.framework.TestCase.assertTrue;
import android.app.Activity;
+import android.hidl.manager.V1_2.IServiceManager;
+import android.os.RemoteException;
import android.util.Pair;
import androidx.test.InstrumentationRegistry;
import androidx.test.filters.LargeTest;
@@ -47,21 +49,30 @@
*/
@RunWith(Parameterized.class)
public class NNAccuracyTest {
- private static final String ARG_HAL_SERVICE_INSTANCE = "halServiceInstance";
- private static final String HAL_SERVICE_INSTANCE_PREFIX =
- "android.hardware.neuralnetworks@1.3::IDevice/";
+ private static final String HAL_SERVICE_TYPE = "android.hardware.neuralnetworks@1.3::IDevice";
@Rule
public ActivityTestRule<NNAccuracyActivity> mActivityRule =
new ActivityTestRule<>(NNAccuracyActivity.class);
- @Parameterized.Parameter(0) public TestModels.TestModelEntry mModel;
+ private static class InstanceModel {
+ final String mInstance;
+ final TestModels.TestModelEntry mEntry;
+
+ InstanceModel(String instance, TestModels.TestModelEntry entry) {
+ mInstance = instance;
+ mEntry = entry;
+ }
+
+ @Override
+ public String toString() {
+ return mInstance + ":" + mEntry.mModelName;
+ }
+ }
+ @Parameterized.Parameter(0) public InstanceModel mModel;
private Activity mActivity;
- private String mHalServiceInstance;
- private String mDeviceName;
-
// TODO(vddang): Add mobilenet_v1_0.25_128_quant_topk_aosp
private static final String[] MODEL_NAMES = new String[] {
"tts_float",
@@ -81,37 +92,42 @@
};
@Parameters(name = "{0}")
- public static List<TestModels.TestModelEntry> modelsList() {
- List<TestModels.TestModelEntry> models = new ArrayList<>();
- for (String modelName : MODEL_NAMES) {
- models.add(TestModels.getModelByName(modelName));
+ public static Iterable<InstanceModel> parameterList() throws RemoteException {
+ List<String> serviceInstances =
+ IServiceManager.getService().listManifestByInterface(HAL_SERVICE_TYPE);
+ List<InstanceModel> parameters = new ArrayList<>();
+ for (String instance : serviceInstances) {
+ for (String modelName : MODEL_NAMES) {
+ TestModels.TestModelEntry entry = TestModels.getModelByName(modelName);
+ if (entry == null) {
+ throw new RuntimeException("Failed to find model of name " + modelName);
+ }
+ parameters.add(new InstanceModel(instance, entry));
+ }
}
- return Collections.unmodifiableList(models);
+
+ return Collections.unmodifiableList(parameters);
}
@Before
public void setUp() throws Exception {
mActivity = mActivityRule.getActivity();
- mHalServiceInstance =
- InstrumentationRegistry.getArguments().getString(ARG_HAL_SERVICE_INSTANCE);
- assertTrue(mHalServiceInstance.startsWith(HAL_SERVICE_INSTANCE_PREFIX));
- mDeviceName = mHalServiceInstance.substring(HAL_SERVICE_INSTANCE_PREFIX.length());
}
@Test
@LargeTest
public void testDriver() throws BenchmarkException, IOException {
- NNTestBase test = mModel.createNNTestBase();
+ NNTestBase test = mModel.mEntry.createNNTestBase();
test.useNNApi();
- test.setNNApiDeviceName(mDeviceName);
+ test.setNNApiDeviceName(mModel.mInstance);
if (!test.setupModel(mActivity)) {
throw new AssumptionViolatedException("The driver rejected the model.");
}
Pair<List<InferenceInOutSequence>, List<InferenceResult>> inferenceResults =
test.runBenchmarkCompleteInputSet(/*setRepeat=*/1, /*timeoutSec=*/3600);
- BenchmarkResult benchmarkResult = BenchmarkResult.fromInferenceResults(mModel.mModelName,
- BenchmarkResult.BACKEND_TFLITE_NNAPI, inferenceResults.first,
- inferenceResults.second, test.getEvaluator());
+ BenchmarkResult benchmarkResult = BenchmarkResult.fromInferenceResults(
+ mModel.mEntry.mModelName, BenchmarkResult.BACKEND_TFLITE_NNAPI,
+ inferenceResults.first, inferenceResults.second, test.getEvaluator());
assertFalse(benchmarkResult.hasValidationErrors());
}
}
diff --git a/tv/OWNERS b/tv/OWNERS
new file mode 100644
index 0000000..a68d120
--- /dev/null
+++ b/tv/OWNERS
@@ -0,0 +1 @@
+include platform/frameworks/base:/media/java/android/media/tv/OWNERS
