IVGCVSW-5377 Add ArmNN TfLite delegate to ExecuteNetwork
* Added package manger to turn internal calls to find_package into a no-op
* Changed delegate cmake so it can now be built within armnn
Change-Id: I2a7ecb9a3c1ca05474cd1dccd91498f6f6c0b32e
Signed-off-by: Finn Williams <Finn.Williams@arm.com>
Signed-off-by: Sadik Armagan <sadik.armagan@arm.com>
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 30b03dc..cee3c2a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -17,6 +17,16 @@
include(${cmake_file})
endforeach()
+cmake_policy(SET CMP0057 NEW)
+
+set(as_subproject Armnn)
+
+macro(find_package)
+ if(NOT ${ARGV0} IN_LIST as_subproject)
+ _find_package(${ARGV})
+ endif()
+endmacro()
+
if (DYNAMIC_BACKEND_PATHS)
# It's expected to have the format: DYNAMIC_BACKEND_PATHS="PATH_1:PATH_2...:PATH_N"
add_definitions('-DDYNAMIC_BACKEND_PATHS="${DYNAMIC_BACKEND_PATHS}"')
@@ -29,6 +39,15 @@
add_subdirectory(src/armnnSerializer)
add_subdirectory(src/armnnDeserializer)
+
+if (BUILD_ARMNN_TFLITE_DELEGATE)
+
+ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/delegate/cmake/Modules)
+ add_subdirectory(delegate)
+
+ add_definitions(-DARMNN_TF_LITE_DELEGATE)
+endif()
+
if (BUILD_TESTS)
add_subdirectory(tests)
endif()
@@ -1164,6 +1183,9 @@
NAMESPACE Armnn::
)
+add_library(Armnn::Armnn ALIAS armnn)
+add_library(Armnn::armnnUtils ALIAS armnnUtils)
+
####################################################
## Build Python bindings
if (BUILD_PYTHON_WHL OR BUILD_PYTHON_SRC)
diff --git a/cmake/GlobalConfig.cmake b/cmake/GlobalConfig.cmake
index 360f068..843ad6b 100644
--- a/cmake/GlobalConfig.cmake
+++ b/cmake/GlobalConfig.cmake
@@ -35,6 +35,7 @@
option(BUILD_PYTHON_SRC "Build Python source package" OFF)
option(BUILD_STATIC_PIPE_LIBS "Build Static PIPE libraries" OFF)
option(BUILD_PIPE_ONLY "Build the PIPE libraries only" OFF)
+option(BUILD_ARMNN_TFLITE_DELEGATE "Build the Arm NN TfLite delegate" OFF)
include(SelectLibraryConfigurations)
@@ -210,6 +211,9 @@
include_directories(SYSTEM "${ONNX_GENERATED_SOURCES}")
endif()
+if(BUILD_ARMNN_TFLITE_DELEGATE)
+ add_definitions(-DARMNN_TFLITE_DELEGATE)
+endif()
# Flatbuffers support for TF Lite and Armnn Serializer
if(BUILD_TF_LITE_PARSER OR BUILD_ARMNN_SERIALIZER)
# verify we have a valid flatbuffers include path
diff --git a/delegate/CMakeLists.txt b/delegate/CMakeLists.txt
index 595784f..d53af88 100644
--- a/delegate/CMakeLists.txt
+++ b/delegate/CMakeLists.txt
@@ -195,5 +195,7 @@
EXPORT armnn-delegate-targets
FILE ${CMAKE_CURRENT_BINARY_DIR}/ArmnnDelegateTargets.cmake
NAMESPACE ArmnnDelegate::)
+add_library(ArmnnDelegate::ArmnnDelegate ALIAS armnnDelegate)
+
####################################################
diff --git a/delegate/cmake/Modules/ArmnnDelegateConfig.cmake.in b/delegate/cmake/Modules/ArmnnDelegateConfig.cmake.in
index 12d1161..c403068 100644
--- a/delegate/cmake/Modules/ArmnnDelegateConfig.cmake.in
+++ b/delegate/cmake/Modules/ArmnnDelegateConfig.cmake.in
@@ -9,6 +9,8 @@
include(CMakeFindDependencyMacro)
+find_dependency(Armnn REQUIRED CONFIG)
+
list(APPEND CMAKE_MODULE_PATH ${ARMNN_DELEGATE_CMAKE_DIR})
if(NOT TARGET ArmnnDelegate::ArmnnDelegate)
diff --git a/delegate/include/armnn_delegate.hpp b/delegate/include/armnn_delegate.hpp
index 6f18185..adf264a 100644
--- a/delegate/include/armnn_delegate.hpp
+++ b/delegate/include/armnn_delegate.hpp
@@ -3,8 +3,7 @@
// SPDX-License-Identifier: MIT
//
-#ifndef ARMNN_TFLITE_DELEGATE
-#define ARMNN_TFLITE_DELEGATE
+#pragma once
#include "DelegateOptions.hpp"
@@ -114,5 +113,4 @@
} // armnnDelegate namespace
-#endif // ARMNN_TFLITE_DELEGATE
diff --git a/delegate/src/Transpose.hpp b/delegate/src/Transpose.hpp
index dd79bd3..f3c348a 100644
--- a/delegate/src/Transpose.hpp
+++ b/delegate/src/Transpose.hpp
@@ -64,7 +64,7 @@
auto* permTensorDataPtr = tflite::GetTensorData<int32_t>(&tfLiteInputTensor1);
unsigned int numEl = tfLiteInputTensor1.dims->data[0];
- ARMNN_ASSERT( numEl <= armnn::MaxNumOfTensorDimensions);
+ ARMNN_ASSERT( numEl <= static_cast<int>(armnn::MaxNumOfTensorDimensions));
ARMNN_ASSERT( tfLiteInputTensor1.dims->size == 1); // ensure only single dimension to the permutation tensor
armnn::TransposeDescriptor descriptor(armnn::PermutationVector(
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 9d3b026..edea34d 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -264,12 +264,15 @@
target_link_libraries(ExecuteNetwork armnnTfLiteParser)
endif()
if (BUILD_ONNX_PARSER)
- target_link_libraries(ExecuteNetwork armnnOnnxParser)
+ target_link_libraries(ExecuteNetwork armnnOnnxParser)
endif()
-
+ if (BUILD_ARMNN_TFLITE_DELEGATE)
+ target_link_libraries(ExecuteNetwork ArmnnDelegate::ArmnnDelegate)
+ endif()
target_link_libraries(ExecuteNetwork armnn)
- target_link_libraries(ExecuteNetwork ${CMAKE_THREAD_LIBS_INIT})
- addDllCopyCommands(ExecuteNetwork)
+
+ target_link_libraries(ExecuteNetwork ${CMAKE_THREAD_LIBS_INIT})
+ addDllCopyCommands(ExecuteNetwork)
endif()
if(BUILD_ACCURACY_TOOL)
diff --git a/tests/ExecuteNetwork/ExecuteNetwork.cpp b/tests/ExecuteNetwork/ExecuteNetwork.cpp
index c17eabd..fa84a6e 100644
--- a/tests/ExecuteNetwork/ExecuteNetwork.cpp
+++ b/tests/ExecuteNetwork/ExecuteNetwork.cpp
@@ -25,9 +25,194 @@
#if defined(ARMNN_ONNX_PARSER)
#include "armnnOnnxParser/IOnnxParser.hpp"
#endif
+#if defined(ARMNN_TFLITE_DELEGATE)
+#include <armnn_delegate.hpp>
+#include <DelegateOptions.hpp>
+
+#include <tensorflow/lite/builtin_ops.h>
+#include <tensorflow/lite/c/builtin_op_data.h>
+#include <tensorflow/lite/c/common.h>
+#include <tensorflow/lite/optional_debug_tools.h>
+#include <tensorflow/lite/kernels/builtin_op_kernels.h>
+#include <tensorflow/lite/interpreter.h>
+#include <tensorflow/lite/kernels/register.h>
+#endif
#include <future>
+#if defined(ARMNN_TFLITE_DELEGATE)
+int TfLiteDelegateMainImpl(const ExecuteNetworkParams& params,
+ const std::shared_ptr<armnn::IRuntime>& runtime = nullptr)
+{
+ using namespace tflite;
+ std::unique_ptr<tflite::FlatBufferModel> model = tflite::FlatBufferModel::BuildFromFile(params.m_ModelPath.c_str());
+
+ auto tfLiteInterpreter = std::make_unique<Interpreter>();
+ tflite::ops::builtin::BuiltinOpResolver resolver;
+
+ tflite::InterpreterBuilder builder(*model, resolver);
+ builder(&tfLiteInterpreter);
+ tfLiteInterpreter->AllocateTensors();
+
+ // Create the Armnn Delegate
+ armnnDelegate::DelegateOptions delegateOptions(params.m_ComputeDevices);
+ std::unique_ptr<TfLiteDelegate, decltype(&armnnDelegate::TfLiteArmnnDelegateDelete)>
+ theArmnnDelegate(armnnDelegate::TfLiteArmnnDelegateCreate(delegateOptions),
+ armnnDelegate::TfLiteArmnnDelegateDelete);
+ // Register armnn_delegate to TfLiteInterpreter
+ int status = tfLiteInterpreter->ModifyGraphWithDelegate(std::move(theArmnnDelegate));
+
+ std::vector<std::string> inputBindings;
+ for (const std::string& inputName: params.m_InputNames)
+ {
+ inputBindings.push_back(inputName);
+ }
+
+ armnn::Optional<std::string> dataFile = params.m_GenerateTensorData
+ ? armnn::EmptyOptional()
+ : armnn::MakeOptional<std::string>(params.m_InputTensorDataFilePaths[0]);
+
+ const size_t numInputs = inputBindings.size();
+
+ for(unsigned int inputIndex = 0; inputIndex < numInputs; ++inputIndex)
+ {
+ int input = tfLiteInterpreter->inputs()[inputIndex];
+ if (params.m_InputTypes[inputIndex].compare("float") == 0)
+ {
+ auto inputData = tfLiteInterpreter->typed_tensor<float>(input);
+ TContainer tensorData;
+ PopulateTensorWithData(tensorData,
+ params.m_InputTensorShapes[inputIndex]->GetNumElements(),
+ params.m_InputTypes[inputIndex],
+ armnn::EmptyOptional(),
+ dataFile);
+ inputData = reinterpret_cast<float*>(&tensorData);
+ armnn::IgnoreUnused(inputData);
+ }
+ else if (params.m_InputTypes[inputIndex].compare("int") == 0)
+ {
+ auto inputData = tfLiteInterpreter->typed_tensor<int32_t>(input);
+ TContainer tensorData;
+ PopulateTensorWithData(tensorData,
+ params.m_InputTensorShapes[inputIndex]->GetNumElements(),
+ params.m_InputTypes[inputIndex],
+ armnn::EmptyOptional(),
+ dataFile);
+ inputData = reinterpret_cast<int32_t*>(&tensorData);
+ armnn::IgnoreUnused(inputData);
+ }
+ else if (params.m_InputTypes[inputIndex].compare("qasymm8") == 0)
+ {
+ auto inputData = tfLiteInterpreter->typed_tensor<uint8_t>(input);
+ TContainer tensorData;
+ PopulateTensorWithData(tensorData,
+ params.m_InputTensorShapes[inputIndex]->GetNumElements(),
+ params.m_InputTypes[inputIndex],
+ armnn::EmptyOptional(),
+ dataFile);
+ inputData = reinterpret_cast<uint8_t*>(&tensorData);
+ armnn::IgnoreUnused(inputData);
+ }
+ else
+ {
+ ARMNN_LOG(fatal) << "Unsupported input tensor data type \"" << params.m_InputTypes[inputIndex] << "\". ";
+ return EXIT_FAILURE;
+ }
+ }
+
+ for (size_t x = 0; x < params.m_Iterations; x++)
+ {
+ // Run the inference
+ tfLiteInterpreter->Invoke();
+
+ // Print out the output
+ for (unsigned int outputIndex = 0; outputIndex < params.m_OutputNames.size(); ++outputIndex)
+ {
+ std::cout << "Printing out the output" << std::endl;
+ auto tfLiteDelegateOutputId = tfLiteInterpreter->outputs()[outputIndex];
+ TfLiteIntArray *outputDims = tfLiteInterpreter->tensor(tfLiteDelegateOutputId)->dims;
+
+ int outputSize = 1;
+ for (unsigned int dim = 0; dim < static_cast<unsigned int>(outputDims->size); ++dim)
+ {
+ outputSize *= outputDims->data[dim];
+ }
+
+ std::cout << params.m_OutputNames[outputIndex] << ": ";
+ if (params.m_OutputTypes[outputIndex].compare("float") == 0)
+ {
+ auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<float>(tfLiteDelegateOutputId);
+
+ if(tfLiteDelageOutputData == NULL)
+ {
+ ARMNN_LOG(fatal) << "Output tensor is null, output type: "
+ "\"" << params.m_OutputTypes[outputIndex] << "\" may be incorrect.";
+ return EXIT_FAILURE;
+ }
+
+ for (int i = 0; i < outputSize; ++i)
+ {
+ std::cout << tfLiteDelageOutputData[i] << ", ";
+ if (i % 60 == 0)
+ {
+ std::cout << std::endl;
+ }
+ }
+ }
+ else if (params.m_OutputTypes[outputIndex].compare("int") == 0)
+ {
+ auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<int32_t>(tfLiteDelegateOutputId);
+
+ if(tfLiteDelageOutputData == NULL)
+ {
+ ARMNN_LOG(fatal) << "Output tensor is null, output type: "
+ "\"" << params.m_OutputTypes[outputIndex] << "\" may be incorrect.";
+ return EXIT_FAILURE;
+ }
+
+ for (int i = 0; i < outputSize; ++i)
+ {
+ std::cout << tfLiteDelageOutputData[i] << ", ";
+ if (i % 60 == 0)
+ {
+ std::cout << std::endl;
+ }
+ }
+ }
+ else if (params.m_OutputTypes[outputIndex].compare("qasymm8") == 0)
+ {
+ auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<uint8_t>(tfLiteDelegateOutputId);
+
+ if(tfLiteDelageOutputData == NULL)
+ {
+ ARMNN_LOG(fatal) << "Output tensor is null, output type: "
+ "\"" << params.m_OutputTypes[outputIndex] << "\" may be incorrect.";
+ return EXIT_FAILURE;
+ }
+
+ for (int i = 0; i < outputSize; ++i)
+ {
+ std::cout << unsigned(tfLiteDelageOutputData[i]) << ", ";
+ if (i % 60 == 0)
+ {
+ std::cout << std::endl;
+ }
+ }
+ }
+ else
+ {
+ ARMNN_LOG(fatal) << "Output tensor is null, output type: "
+ "\"" << params.m_OutputTypes[outputIndex] <<
+ "\" may be incorrect. Output type can be specified with -z argument";
+ return EXIT_FAILURE;
+ }
+ std::cout << std::endl;
+ }
+ }
+
+ return status;
+}
+#endif
template<typename TParser, typename TDataType>
int MainImpl(const ExecuteNetworkParams& params,
const std::shared_ptr<armnn::IRuntime>& runtime = nullptr)
@@ -242,6 +427,16 @@
}
else if(modelFormat.find("tflite") != std::string::npos)
{
+
+ if (ProgramOptions.m_ExNetParams.m_EnableDelegate)
+ {
+ #if defined(ARMNN_TF_LITE_DELEGATE)
+ return TfLiteDelegateMainImpl(ProgramOptions.m_ExNetParams, runtime);
+ #else
+ ARMNN_LOG(fatal) << "Not built with Tensorflow-Lite parser support.";
+ return EXIT_FAILURE;
+ #endif
+ }
#if defined(ARMNN_TF_LITE_PARSER)
return MainImpl<armnnTfLiteParser::ITfLiteParser, float>(ProgramOptions.m_ExNetParams, runtime);
#else
diff --git a/tests/ExecuteNetwork/ExecuteNetworkParams.cpp b/tests/ExecuteNetwork/ExecuteNetworkParams.cpp
index a3a7f6a..890ab2a 100644
--- a/tests/ExecuteNetwork/ExecuteNetworkParams.cpp
+++ b/tests/ExecuteNetwork/ExecuteNetworkParams.cpp
@@ -74,6 +74,7 @@
"format supported for tflite files",
modelFormat));
}
+#elif defined(ARMNN_TFLITE_DELEGATE)
#else
throw armnn::InvalidArgumentException("Can't run model in tflite format without a "
"built with Tensorflow Lite parser support.");
diff --git a/tests/ExecuteNetwork/ExecuteNetworkParams.hpp b/tests/ExecuteNetwork/ExecuteNetworkParams.hpp
index 5490230..8f176c2 100644
--- a/tests/ExecuteNetwork/ExecuteNetworkParams.hpp
+++ b/tests/ExecuteNetwork/ExecuteNetworkParams.hpp
@@ -24,6 +24,7 @@
bool m_EnableProfiling;
bool m_GenerateTensorData;
bool m_InferOutputShape = false;
+ bool m_EnableDelegate = false;
std::vector<std::string> m_InputNames;
std::vector<std::string> m_InputTensorDataFilePaths;
std::vector<TensorShapePtr> m_InputTensorShapes;
diff --git a/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp b/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
index e37c4eb..b499289 100644
--- a/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
+++ b/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
@@ -168,6 +168,10 @@
"tensorflow-text.",
cxxopts::value<std::string>())
+ ("D,armnn-tflite-delegate",
+ "enable Arm NN TfLite delegate",
+ cxxopts::value<bool>(m_ExNetParams.m_EnableDelegate)->default_value("false")->implicit_value("true"))
+
("m,model-path",
"Path to model file, e.g. .armnn, .caffemodel, .prototxt, .tflite, .onnx",
cxxopts::value<std::string>(m_ExNetParams.m_ModelPath))
