Integrate NNAPI unit tests into CTS
This CL wraps existing NNAPI gtest-based unit tests as a CTS test. It
changes the previous JNI-based CTS app to a native C++ test.
Bug: 63905942
Test: cts-tradefed run commandAndExit cts -m CtsNNAPITestCases
Change-Id: I97427d78dc0fa2c9adf7e3bb2ec784d599353fbb
diff --git a/tests/tests/neuralnetworks/Android.mk b/tests/tests/neuralnetworks/Android.mk
index 330fde4..cb0c822 100644
--- a/tests/tests/neuralnetworks/Android.mk
+++ b/tests/tests/neuralnetworks/Android.mk
@@ -12,31 +12,44 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Build the unit tests.
+
LOCAL_PATH:= $(call my-dir)
include $(CLEAR_VARS)
-
-# Replace "Example" with your name.
-LOCAL_PACKAGE_NAME := CtsNNAPITestCases
-
-# Don't include this package in any target.
-LOCAL_MODULE_TAGS := optional
-
-# Include both the 32 and 64 bit versions
+LOCAL_MODULE := CtsNNAPITestCases
+LOCAL_MODULE_PATH := $(TARGET_OUT_DATA)/nativetest
LOCAL_MULTILIB := both
+LOCAL_MODULE_STEM_32 := $(LOCAL_MODULE)32
+LOCAL_MODULE_STEM_64 := $(LOCAL_MODULE)64
-# When built, explicitly put it in the data partition.
-LOCAL_MODULE_PATH := $(TARGET_OUT_DATA_APPS)
+LOCAL_SRC_FILES := \
+ src/TestValidation.cpp \
+ src/TestMemory.cpp \
+ src/TestTrivialModel.cpp \
+ src/TestGenerated.cpp
-LOCAL_STATIC_JAVA_LIBRARIES := ctstestrunner legacy-android-test
-LOCAL_JNI_SHARED_LIBRARIES := libnnapitest_jni
+LOCAL_C_INCLUDES := frameworks/ml/nn/runtime/include/
+LOCAL_C_INCLUDES += frameworks/ml/nn/runtime/test/
+LOCAL_C_INCLUDES += frameworks/ml/nn/runtime/
+LOCAL_C_INCLUDES += frameworks/ml/nn/common/include
+LOCAL_C_INCLUDES += frameworks/ml/nn/tools/test_generator/include
-LOCAL_SRC_FILES := $(call all-java-files-under, src)
+LOCAL_SHARED_LIBRARIES := liblog libneuralnetworks
-LOCAL_SDK_VERSION := current
+# TODO: use the libgtest_ndk_c++ instead
+LOCAL_STATIC_LIBRARIES := libgtest libgtest_main
+
+LOCAL_CTS_TEST_PACKAGE := android.neuralnetworks
# Tag this module as a cts test artifact
LOCAL_COMPATIBILITY_SUITE := cts vts general-tests
-include $(BUILD_CTS_PACKAGE)
-include $(LOCAL_PATH)/libnnapitest/Android.mk
+LOCAL_CFLAGS := -Werror -Wall
+
+# TODO: use the following two lines instead
+#LOCAL_SDK_VERSION := current
+#LOCAL_NDK_STL_VARIANT := c++_static
+LOCAL_CXX_STL := libc++
+
+include $(BUILD_CTS_EXECUTABLE)
diff --git a/tests/tests/neuralnetworks/AndroidManifest.xml b/tests/tests/neuralnetworks/AndroidManifest.xml
deleted file mode 100644
index 8b59a58..0000000
--- a/tests/tests/neuralnetworks/AndroidManifest.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Copyright (C) 2017 The Android Open Source Project
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<manifest xmlns:android="http://schemas.android.com/apk/res/android"
- package="android.nnapi.cts">
-
- <uses-permission android:name="android.permission.DISABLE_KEYGUARD" />
- <application>
- <uses-library android:name="android.test.runner" />
- </application>
-
- <!-- This is a self-instrumenting test package. -->
- <instrumentation android:name="android.support.test.runner.AndroidJUnitRunner"
- android:targetPackage="android.nnapi.cts"
- android:label="CTS tests of NNAPI">
- <meta-data android:name="listener"
- android:value="com.android.cts.runner.CtsTestRunListener" />
- </instrumentation>
-
-</manifest>
-
diff --git a/tests/tests/neuralnetworks/AndroidTest.xml b/tests/tests/neuralnetworks/AndroidTest.xml
index 5d253b2..cd1a0f0 100644
--- a/tests/tests/neuralnetworks/AndroidTest.xml
+++ b/tests/tests/neuralnetworks/AndroidTest.xml
@@ -13,15 +13,18 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<configuration description="Configuration for NNAPI Tests">
+<configuration description="Configuration for Native NNAPI Tests">
<option name="config-descriptor:metadata" key="component" value="neuralnetworks" />
- <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
- <option name="cleanup-apks" value="true" />
- <option name="test-file-name" value="CtsNNAPITestCases.apk" />
+ <target_preparer class="com.android.compatibility.common.tradefed.targetprep.FilePusher">
+ <option name="cleanup" value="true" />
+ <option name="push" value="CtsNNAPITestCases->/data/local/tmp/CtsNNAPITestCases" />
+ <option name="append-bitness" value="true" />
</target_preparer>
-
- <test class="com.android.tradefed.testtype.AndroidJUnitTest" >
- <option name="package" value="android.nnapi.cts" />
- <option name="runtime-hint" value="5m30s" />
+ <test class="com.android.tradefed.testtype.GTest" >
+ <option name="native-test-device-path" value="/data/local/tmp" />
+ <option name="module-name" value="CtsNNAPITestCases" />
+ <option name="runtime-hint" value="2m" />
+ <!-- test-timeout unit is ms, value = 2 min -->
+ <option name="native-test-timeout" value="120000" />
</test>
</configuration>
diff --git a/tests/tests/neuralnetworks/libnnapitest/Android.mk b/tests/tests/neuralnetworks/libnnapitest/Android.mk
deleted file mode 100644
index e0f456f..0000000
--- a/tests/tests/neuralnetworks/libnnapitest/Android.mk
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright (C) 2017 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# This is the shared library included by the JNI test app.
-#
-LOCAL_PATH := $(call my-dir)
-include $(CLEAR_VARS)
-LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk
-LOCAL_CLANG := true
-LOCAL_MODULE := libnnapitest_jni
-LOCAL_MODULE_TAGS := optional
-
-LOCAL_SRC_FILES := \
- test_generated.cpp
-
-LOCAL_C_INCLUDES := $(JNI_H_INCLUDE)
-LOCAL_C_INCLUDES += frameworks/ml/nn/runtime/include/
-LOCAL_C_INCLUDES += frameworks/ml/nn/runtime/test/
-
-LOCAL_CPPFLAGS := -std=c++11
-LOCAL_CFLAGS := -Wno-unused-parameter
-
-LOCAL_SHARED_LIBRARIES := libdl liblog libneuralnetworks
-
-# TODO: specify LOCAL_SDK_VERSION when libneuralnetworks become available
-# in NDK
-#LOCAL_SDK_VERSION := 27
-
-LOCAL_CXX_STL := libc++_static
-
-include $(BUILD_SHARED_LIBRARY)
-
-
-
diff --git a/tests/tests/neuralnetworks/libnnapitest/test_generated.cpp b/tests/tests/neuralnetworks/libnnapitest/test_generated.cpp
deleted file mode 100644
index 75a9039..0000000
--- a/tests/tests/neuralnetworks/libnnapitest/test_generated.cpp
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Copyright (C) 2017 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Top level driver for models and examples generated by test_generator.py
-
-#include <android/log.h>
-#include <jni.h>
-
-#include <NeuralNetworksWrapper.h>
-
-#include <cmath>
-#include <functional>
-#include <iostream>
-#include <map>
-#include <sstream>
-
-#define LOG_TAG "nnapi"
-#define LOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, LOG_TAG, __VA_ARGS__)
-
-using namespace android::nn::wrapper;
-namespace GeneratedTest {
-template <typename T>
-class Example {
- public:
- typedef T ElementType;
- typedef std::pair<std::map<int, std::vector<T>>,
- std::map<int, std::vector<T>>>
- ExampleType;
-
- static bool Execute(std::function<void(Model*)> create_model,
- std::vector<ExampleType>& examples,
- std::function<bool(const T, const T)> compare) {
- android::nn::wrapper::Initialize();
- Model model;
- create_model(&model);
-
- int example_no = 1;
- bool error = false;
-
- for (auto& example : examples) {
- Request request(&model);
-
- // Go through all inputs
- for (auto& i : example.first) {
- std::vector<T>& input = i.second;
- request.setInput(i.first, (const void*)input.data(),
- input.size() * sizeof(T));
- }
-
- std::map<int, std::vector<T>> test_outputs;
-
- if (example.second.size() != 1) {
- LOGV("nnapi: output > 1 probably will not work");
- }
- int output_no = 0;
- for (auto& i : example.second) {
- std::vector<T>& output = i.second;
- test_outputs[i.first].resize(output.size());
- std::vector<T>& test_output = test_outputs[i.first];
- request.setOutput(output_no++, (void*)test_output.data(),
- test_output.size() * sizeof(T));
- }
- Result r = request.compute();
- if (r != Result::NO_ERROR)
- LOGV("Request was not completed normally");
- bool mismatch = false;
- for (auto& i : example.second) {
- const std::vector<T>& test = test_outputs[i.first];
- const std::vector<T>& golden = i.second;
- for (unsigned i = 0; i < golden.size(); i++) {
- if (compare(golden[i], test[i])) {
- std::stringstream ss;
- ss << " output[" << i << "] = " << (float)test[i]
- << " (should be " << (float)golden[i] << ")\n";
- LOGV("%s", ss.str().c_str());
- error = error || true;
- mismatch = mismatch || true;
- }
- }
- }
- if (mismatch) {
- std::stringstream ss;
- ss << "Example: " << example_no++;
- ss << " failed\n";
- LOGV("%s", ss.str().c_str());
- }
- }
-
- android::nn::wrapper::Shutdown();
- return error;
- }
-};
-}; // namespace GeneratedTest
-
-// Float32 examples
-typedef GeneratedTest::Example<float>::ExampleType Example;
-// Quantized examples
-typedef GeneratedTest::Example<uint8_t>::ExampleType QExample;
-#define assert(x)
-namespace avg_pool_quant8 {
-std::vector<QExample> examples = {
-// Generated avg_pool quantized test
-#include "generated/examples/avg_pool_quant8_tests.example.cc"
-};
-// Generated model constructor
-#include "generated/models/avg_pool_quant8.model.cpp"
-} // namespace avg_pool_quant8
-
-namespace conv_1_h3_w2_SAME {
-std::vector<Example> examples = {
-// Converted examples
-#include "generated/examples/conv_1_h3_w2_SAME_tests.example.cc"
-};
-// Generated model constructor
-#include "generated/models/conv_1_h3_w2_SAME.model.cpp"
-} // namespace conv_1_h3_w2_SAME
-
-namespace mobilenet {
-std::vector<Example> examples = {
-// Converted examples
-#include "generated/examples/mobilenet_224_gender_basic_fixed_tests.example.cc"
-};
-// Generated model constructor
-#include "generated/models/mobilenet_224_gender_basic_fixed.model.cpp"
-} // namespace mobilenet
-
-namespace {
-bool QExecute(std::function<void(Model*)> create_model,
- std::vector<QExample>& examples) {
- return GeneratedTest::Example<uint8_t>::Execute(
- create_model, examples,
- [](uint8_t golden, uint8_t test) { return golden != test; });
-}
-
-bool Execute(std::function<void(Model*)> create_model,
- std::vector<Example>& examples) {
- return GeneratedTest::Example<float>::Execute(
- create_model, examples, [](float golden, float test) {
- return std::fabs(golden - test) > 1.5e-5f;
- });
-}
-
-} // namespace
-extern "C" JNIEXPORT jboolean JNICALL
-Java_android_cts_nnapi_NNAPIGeneratedTests_avgPoolQuant8(JNIEnv*, jclass) {
- return QExecute(avg_pool_quant8::CreateModel, avg_pool_quant8::examples);
-}
-
-extern "C" JNIEXPORT jboolean JNICALL
-Java_android_cts_nnapi_NNAPIGeneratedTests_conv1H3W2Same(JNIEnv*, jclass) {
- return Execute(conv_1_h3_w2_SAME::CreateModel, conv_1_h3_w2_SAME::examples);
-}
-
-extern "C" JNIEXPORT jboolean JNICALL
-Java_android_cts_nnapi_NNAPIGeneratedTests_mobileNet(JNIEnv*, jclass) {
- return Execute(mobilenet::CreateModel, mobilenet::examples);
-}
diff --git a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java b/tests/tests/neuralnetworks/src/TestGenerated.cpp
similarity index 66%
copy from tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java
copy to tests/tests/neuralnetworks/src/TestGenerated.cpp
index db37a8f646c..b9044fe 100644
--- a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java
+++ b/tests/tests/neuralnetworks/src/TestGenerated.cpp
@@ -14,19 +14,5 @@
* limitations under the License.
*/
-package android.cts.nnapi;
-
-import android.test.AndroidTestCase;
-
-public class NNAPITest extends AndroidTestCase {
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- }
-
- @Override
- protected void tearDown() throws Exception {
- super.tearDown();
- }
-}
+// Include corresponding NNAPI unit tests in frameworks/ml/nn/runtime/test
+#include "test/TestGenerated.cpp"
diff --git a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java b/tests/tests/neuralnetworks/src/TestMemory.cpp
similarity index 66%
rename from tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java
rename to tests/tests/neuralnetworks/src/TestMemory.cpp
index db37a8f646c..84d2d41 100644
--- a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java
+++ b/tests/tests/neuralnetworks/src/TestMemory.cpp
@@ -14,19 +14,5 @@
* limitations under the License.
*/
-package android.cts.nnapi;
-
-import android.test.AndroidTestCase;
-
-public class NNAPITest extends AndroidTestCase {
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- }
-
- @Override
- protected void tearDown() throws Exception {
- super.tearDown();
- }
-}
+// Include corresponding NNAPI unit tests in frameworks/ml/nn/runtime/test
+#include "test/TestMemory.cpp"
diff --git a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java b/tests/tests/neuralnetworks/src/TestTrivialModel.cpp
similarity index 66%
copy from tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java
copy to tests/tests/neuralnetworks/src/TestTrivialModel.cpp
index db37a8f646c..b41e50f 100644
--- a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java
+++ b/tests/tests/neuralnetworks/src/TestTrivialModel.cpp
@@ -14,19 +14,5 @@
* limitations under the License.
*/
-package android.cts.nnapi;
-
-import android.test.AndroidTestCase;
-
-public class NNAPITest extends AndroidTestCase {
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- }
-
- @Override
- protected void tearDown() throws Exception {
- super.tearDown();
- }
-}
+// Include corresponding NNAPI unit tests in frameworks/ml/nn/runtime/test
+#include "test/TestTrivialModel.cpp"
diff --git a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java b/tests/tests/neuralnetworks/src/TestValidation.cpp
similarity index 66%
copy from tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java
copy to tests/tests/neuralnetworks/src/TestValidation.cpp
index db37a8f646c..eab0f59 100644
--- a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPITest.java
+++ b/tests/tests/neuralnetworks/src/TestValidation.cpp
@@ -14,19 +14,5 @@
* limitations under the License.
*/
-package android.cts.nnapi;
-
-import android.test.AndroidTestCase;
-
-public class NNAPITest extends AndroidTestCase {
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- }
-
- @Override
- protected void tearDown() throws Exception {
- super.tearDown();
- }
-}
+// Include corresponding NNAPI unit tests in frameworks/ml/nn/runtime/test
+#include "test/TestValidation.cpp"
diff --git a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPIGeneratedTests.java b/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPIGeneratedTests.java
deleted file mode 100644
index 646f2b0..0000000
--- a/tests/tests/neuralnetworks/src/android/cts/nnapi/NNAPIGeneratedTests.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (C) 2017 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package android.cts.nnapi;
-
-import android.test.AndroidTestCase;
-import android.util.Log;
-
-public class NNAPIGeneratedTests extends NNAPITest {
- static {
- System.loadLibrary("nnapitest_jni");
- }
-
- native boolean avgPoolQuant8();
- public void testAvgPoolQuant8() {
- assertFalse(avgPoolQuant8());
- }
-
- native boolean conv1H3W2Same();
- public void testConv1H3W2Same() {
- assertFalse(conv1H3W2Same());
- }
-
- native boolean mobileNet();
- public void testMobileNet() {
- assertFalse(mobileNet());
- }
-}
