tests/InferenceTest.inl - platform/external/armnn - Git at Google

 //
 // Copyright © 2017 Arm Ltd. All rights reserved.
 // SPDX-License-Identifier: MIT
 //
 #include "InferenceTest.hpp"

 #include <boost/algorithm/string.hpp>
 #include <boost/numeric/conversion/cast.hpp>
 #include <boost/filesystem/path.hpp>
 #include <boost/assert.hpp>
 #include <boost/format.hpp>
 #include <boost/program_options.hpp>
 #include <boost/filesystem/operations.hpp>

 #include <fstream>
 #include <iostream>
 #include <iomanip>
 #include <array>
 #include <chrono>

 using namespace std;
 using namespace std::chrono;
 using namespace armnn::test;

 namespace armnn
 {
 namespace test
 {

 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;

 template <typename TTestCaseDatabase, typename TModel>
 ClassifierTestCase<TTestCaseDatabase, TModel>::ClassifierTestCase(
     int& numInferencesRef,
     int& numCorrectInferencesRef,
     const std::vector<unsigned int>& validationPredictions,
     std::vector<unsigned int>* validationPredictionsOut,
     TModel& model,
     unsigned int testCaseId,
     unsigned int label,
     std::vector<typename TModel::DataType> modelInput)
     : InferenceModelTestCase<TModel>(
             model, testCaseId, std::vector<TContainer>{ modelInput }, { model.GetOutputSize() })
     , m_Label(label)
     , m_QuantizationParams(model.GetQuantizationParams())
     , m_NumInferencesRef(numInferencesRef)
     , m_NumCorrectInferencesRef(numCorrectInferencesRef)
     , m_ValidationPredictions(validationPredictions)
     , m_ValidationPredictionsOut(validationPredictionsOut)
 {
 }

 struct ClassifierResultProcessor : public boost::static_visitor<>
 {
     using ResultMap = std::map<float,int>;

     ClassifierResultProcessor(float scale, int offset)
         : m_Scale(scale)
         , m_Offset(offset)
     {}

     void operator()(const std::vector<float>& values)
     {
         SortPredictions(values, [](float value)
                                 {
                                     return value;
                                 });
     }

     void operator()(const std::vector<uint8_t>& values)
     {
         auto& scale = m_Scale;
         auto& offset = m_Offset;
         SortPredictions(values, [&scale, &offset](uint8_t value)
                                 {
                                     return armnn::Dequantize(value, scale, offset);
                                 });
     }

     void operator()(const std::vector<int>& values)
     {
         BOOST_ASSERT_MSG(false, "Non-float predictions output not supported.");
     }

     ResultMap& GetResultMap() { return m_ResultMap; }

 private:
     template<typename Container, typename Delegate>
     void SortPredictions(const Container& c, Delegate delegate)
     {
         int index = 0;
         for (const auto& value : c)
         {
             int classification = index++;
             // Take the first class with each probability
             // This avoids strange results when looping over batched results produced
             // with identical test data.
             ResultMap::iterator lb = m_ResultMap.lower_bound(value);

             if (lb == m_ResultMap.end() || !m_ResultMap.key_comp()(value, lb->first))
             {
                 // If the key is not already in the map, insert it.
                 m_ResultMap.insert(lb, ResultMap::value_type(delegate(value), classification));
             }
         }
     }

     ResultMap m_ResultMap;

     float m_Scale=0.0f;
     int m_Offset=0;
 };

 template <typename TTestCaseDatabase, typename TModel>
 TestCaseResult ClassifierTestCase<TTestCaseDatabase, TModel>::ProcessResult(const InferenceTestOptions& params)
 {
     auto& output = this->GetOutputs()[0];
     const auto testCaseId = this->GetTestCaseId();

     ClassifierResultProcessor resultProcessor(m_QuantizationParams.first, m_QuantizationParams.second);
     boost::apply_visitor(resultProcessor, output);

     ARMNN_LOG(info) << "= Prediction values for test #" << testCaseId;
     auto it = resultProcessor.GetResultMap().rbegin();
     for (int i=0; i<5 && it != resultProcessor.GetResultMap().rend(); ++i)
     {
         ARMNN_LOG(info) << "Top(" << (i+1) << ") prediction is " << it->second <<
           " with value: " << (it->first);
         ++it;
     }

     unsigned int prediction = 0;
     boost::apply_visitor([&](auto&& value)
                          {
                              prediction = boost::numeric_cast<unsigned int>(
                                      std::distance(value.begin(), std::max_element(value.begin(), value.end())));
                          },
                          output);

     // If we're just running the defaultTestCaseIds, each one must be classified correctly.
     if (params.m_IterationCount == 0 && prediction != m_Label)
     {
         ARMNN_LOG(error) << "Prediction for test case " << testCaseId << " (" << prediction << ")" <<
             " is incorrect (should be " << m_Label << ")";
         return TestCaseResult::Failed;
     }

     // If a validation file was provided as input, it checks that the prediction matches.
     if (!m_ValidationPredictions.empty() && prediction != m_ValidationPredictions[testCaseId])
     {
         ARMNN_LOG(error) << "Prediction for test case " << testCaseId << " (" << prediction << ")" <<
             " doesn't match the prediction in the validation file (" << m_ValidationPredictions[testCaseId] << ")";
         return TestCaseResult::Failed;
     }

     // If a validation file was requested as output, it stores the predictions.
     if (m_ValidationPredictionsOut)
     {
         m_ValidationPredictionsOut->push_back(prediction);
     }

     // Updates accuracy stats.
     m_NumInferencesRef++;
     if (prediction == m_Label)
     {
         m_NumCorrectInferencesRef++;
     }

     return TestCaseResult::Ok;
 }

 template <typename TDatabase, typename InferenceModel>
 template <typename TConstructDatabaseCallable, typename TConstructModelCallable>
 ClassifierTestCaseProvider<TDatabase, InferenceModel>::ClassifierTestCaseProvider(
     TConstructDatabaseCallable constructDatabase, TConstructModelCallable constructModel)
     : m_ConstructModel(constructModel)
     , m_ConstructDatabase(constructDatabase)
     , m_NumInferences(0)
     , m_NumCorrectInferences(0)
 {
 }

 template <typename TDatabase, typename InferenceModel>
 void ClassifierTestCaseProvider<TDatabase, InferenceModel>::AddCommandLineOptions(
     boost::program_options::options_description& options)
 {
     namespace po = boost::program_options;

     options.add_options()
         ("validation-file-in", po::value<std::string>(&m_ValidationFileIn)->default_value(""),
             "Reads expected predictions from the given file and confirms they match the actual predictions.")
         ("validation-file-out", po::value<std::string>(&m_ValidationFileOut)->default_value(""),
             "Predictions are saved to the given file for later use via --validation-file-in.")
         ("data-dir,d", po::value<std::string>(&m_DataDir)->required(),
             "Path to directory containing test data");

     InferenceModel::AddCommandLineOptions(options, m_ModelCommandLineOptions);
 }

 template <typename TDatabase, typename InferenceModel>
 bool ClassifierTestCaseProvider<TDatabase, InferenceModel>::ProcessCommandLineOptions(
         const InferenceTestOptions& commonOptions)
 {
     if (!ValidateDirectory(m_DataDir))
     {
         return false;
     }

     ReadPredictions();

     m_Model = m_ConstructModel(commonOptions, m_ModelCommandLineOptions);
     if (!m_Model)
     {
         return false;
     }

     m_Database = std::make_unique<TDatabase>(m_ConstructDatabase(m_DataDir.c_str(), *m_Model));
     if (!m_Database)
     {
         return false;
     }

     return true;
 }

 template <typename TDatabase, typename InferenceModel>
 std::unique_ptr<IInferenceTestCase>
 ClassifierTestCaseProvider<TDatabase, InferenceModel>::GetTestCase(unsigned int testCaseId)
 {
     std::unique_ptr<typename TDatabase::TTestCaseData> testCaseData = m_Database->GetTestCaseData(testCaseId);
     if (testCaseData == nullptr)
     {
         return nullptr;
     }

     return std::make_unique<ClassifierTestCase<TDatabase, InferenceModel>>(
         m_NumInferences,
         m_NumCorrectInferences,
         m_ValidationPredictions,
         m_ValidationFileOut.empty() ? nullptr : &m_ValidationPredictionsOut,
         *m_Model,
         testCaseId,
         testCaseData->m_Label,
         std::move(testCaseData->m_InputImage));
 }

 template <typename TDatabase, typename InferenceModel>
 bool ClassifierTestCaseProvider<TDatabase, InferenceModel>::OnInferenceTestFinished()
 {
     const double accuracy = boost::numeric_cast<double>(m_NumCorrectInferences) /
         boost::numeric_cast<double>(m_NumInferences);
     ARMNN_LOG(info) << std::fixed << std::setprecision(3) << "Overall accuracy: " << accuracy;

     // If a validation file was requested as output, the predictions are saved to it.
     if (!m_ValidationFileOut.empty())
     {
         std::ofstream validationFileOut(m_ValidationFileOut.c_str(), std::ios_base::trunc | std::ios_base::out);
         if (validationFileOut.good())
         {
             for (const unsigned int prediction : m_ValidationPredictionsOut)
             {
                 validationFileOut << prediction << std::endl;
             }
         }
         else
         {
             ARMNN_LOG(error) << "Failed to open output validation file: " << m_ValidationFileOut;
             return false;
         }
     }

     return true;
 }

 template <typename TDatabase, typename InferenceModel>
 void ClassifierTestCaseProvider<TDatabase, InferenceModel>::ReadPredictions()
 {
     // Reads the expected predictions from the input validation file (if provided).
     if (!m_ValidationFileIn.empty())
     {
         std::ifstream validationFileIn(m_ValidationFileIn.c_str(), std::ios_base::in);
         if (validationFileIn.good())
         {
             while (!validationFileIn.eof())
             {
                 unsigned int i;
                 validationFileIn >> i;
                 m_ValidationPredictions.emplace_back(i);
             }
         }
         else
         {
             throw armnn::Exception(boost::str(boost::format("Failed to open input validation file: %1%")
                 % m_ValidationFileIn));
         }
     }
 }

 template<typename TConstructTestCaseProvider>
 int InferenceTestMain(int argc,
     char* argv[],
     const std::vector<unsigned int>& defaultTestCaseIds,
     TConstructTestCaseProvider constructTestCaseProvider)
 {
     // Configures logging for both the ARMNN library and this test program.
 #ifdef NDEBUG
     armnn::LogSeverity level = armnn::LogSeverity::Info;
 #else
     armnn::LogSeverity level = armnn::LogSeverity::Debug;
 #endif
     armnn::ConfigureLogging(true, true, level);

     try
     {
         std::unique_ptr<IInferenceTestCaseProvider> testCaseProvider = constructTestCaseProvider();
         if (!testCaseProvider)
         {
             return 1;
         }

         InferenceTestOptions inferenceTestOptions;
         if (!ParseCommandLine(argc, argv, *testCaseProvider, inferenceTestOptions))
         {
             return 1;
         }

         const bool success = InferenceTest(inferenceTestOptions, defaultTestCaseIds, *testCaseProvider);
         return success ? 0 : 1;
     }
     catch (armnn::Exception const& e)
     {
         ARMNN_LOG(fatal) << "Armnn Error: " << e.what();
         return 1;
     }
 }

 //
 // This function allows us to create a classifier inference test based on:
 //  - a model file name
 //  - which can be a binary or a text file for protobuf formats
 //  - an input tensor name
 //  - an output tensor name
 //  - a set of test case ids
 //  - a callback method which creates an object that can return images
 //    called 'Database' in these tests
 //  - and an input tensor shape
 //
 template<typename TDatabase,
          typename TParser,
          typename TConstructDatabaseCallable>
 int ClassifierInferenceTestMain(int argc,
                                 char* argv[],
                                 const char* modelFilename,
                                 bool isModelBinary,
                                 const char* inputBindingName,
                                 const char* outputBindingName,
                                 const std::vector<unsigned int>& defaultTestCaseIds,
                                 TConstructDatabaseCallable constructDatabase,
                                 const armnn::TensorShape* inputTensorShape)

 {
     BOOST_ASSERT(modelFilename);
     BOOST_ASSERT(inputBindingName);
     BOOST_ASSERT(outputBindingName);

     return InferenceTestMain(argc, argv, defaultTestCaseIds,
         [=]
         ()
         {
             using InferenceModel = InferenceModel<TParser, typename TDatabase::DataType>;
             using TestCaseProvider = ClassifierTestCaseProvider<TDatabase, InferenceModel>;

             return make_unique<TestCaseProvider>(constructDatabase,
                 [&]
                 (const InferenceTestOptions &commonOptions,
                  typename InferenceModel::CommandLineOptions modelOptions)
                 {
                     if (!ValidateDirectory(modelOptions.m_ModelDir))
                     {
                         return std::unique_ptr<InferenceModel>();
                     }

                     typename InferenceModel::Params modelParams;
                     modelParams.m_ModelPath = modelOptions.m_ModelDir + modelFilename;
                     modelParams.m_InputBindings  = { inputBindingName };
                     modelParams.m_OutputBindings = { outputBindingName };

                     if (inputTensorShape)
                     {
                         modelParams.m_InputShapes.push_back(*inputTensorShape);
                     }

                     modelParams.m_IsModelBinary = isModelBinary;
                     modelParams.m_ComputeDevices = modelOptions.GetComputeDevicesAsBackendIds();
                     modelParams.m_VisualizePostOptimizationModel = modelOptions.m_VisualizePostOptimizationModel;
                     modelParams.m_EnableFp16TurboMode = modelOptions.m_EnableFp16TurboMode;

                     return std::make_unique<InferenceModel>(modelParams,
                                                             commonOptions.m_EnableProfiling,
                                                             commonOptions.m_DynamicBackendsPath);
             });
         });
 }

 } // namespace test
 } // namespace armnn
	//
	// Copyright © 2017 Arm Ltd. All rights reserved.
	// SPDX-License-Identifier: MIT
	//
	#include "InferenceTest.hpp"

	#include <boost/algorithm/string.hpp>
	#include <boost/numeric/conversion/cast.hpp>
	#include <boost/filesystem/path.hpp>
	#include <boost/assert.hpp>
	#include <boost/format.hpp>
	#include <boost/program_options.hpp>
	#include <boost/filesystem/operations.hpp>

	#include <fstream>
	#include <iostream>
	#include <iomanip>
	#include <array>
	#include <chrono>

	using namespace std;
	using namespace std::chrono;
	using namespace armnn::test;

	namespace armnn
	{
	namespace test
	{

	using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;

	template <typename TTestCaseDatabase, typename TModel>
	ClassifierTestCase<TTestCaseDatabase, TModel>::ClassifierTestCase(
	int& numInferencesRef,
	int& numCorrectInferencesRef,
	const std::vector<unsigned int>& validationPredictions,
	std::vector<unsigned int>* validationPredictionsOut,
	TModel& model,
	unsigned int testCaseId,
	unsigned int label,
	std::vector<typename TModel::DataType> modelInput)
	: InferenceModelTestCase<TModel>(
	model, testCaseId, std::vector<TContainer>{ modelInput }, { model.GetOutputSize() })
	, m_Label(label)
	, m_QuantizationParams(model.GetQuantizationParams())
	, m_NumInferencesRef(numInferencesRef)
	, m_NumCorrectInferencesRef(numCorrectInferencesRef)
	, m_ValidationPredictions(validationPredictions)
	, m_ValidationPredictionsOut(validationPredictionsOut)
	{
	}

	struct ClassifierResultProcessor : public boost::static_visitor<>
	{
	using ResultMap = std::map<float,int>;

	ClassifierResultProcessor(float scale, int offset)
	: m_Scale(scale)
	, m_Offset(offset)
	{}

	void operator()(const std::vector<float>& values)
	{
	SortPredictions(values, [](float value)
	{
	return value;
	});
	}

	void operator()(const std::vector<uint8_t>& values)
	{
	auto& scale = m_Scale;
	auto& offset = m_Offset;
	SortPredictions(values, [&scale, &offset](uint8_t value)
	{
	return armnn::Dequantize(value, scale, offset);
	});
	}

	void operator()(const std::vector<int>& values)
	{
	BOOST_ASSERT_MSG(false, "Non-float predictions output not supported.");
	}

	ResultMap& GetResultMap() { return m_ResultMap; }

	private:
	template<typename Container, typename Delegate>
	void SortPredictions(const Container& c, Delegate delegate)
	{
	int index = 0;
	for (const auto& value : c)
	{
	int classification = index++;
	// Take the first class with each probability
	// This avoids strange results when looping over batched results produced
	// with identical test data.
	ResultMap::iterator lb = m_ResultMap.lower_bound(value);

	if (lb == m_ResultMap.end() \|\| !m_ResultMap.key_comp()(value, lb->first))
	{
	// If the key is not already in the map, insert it.
	m_ResultMap.insert(lb, ResultMap::value_type(delegate(value), classification));
	}
	}
	}

	ResultMap m_ResultMap;

	float m_Scale=0.0f;
	int m_Offset=0;
	};

	template <typename TTestCaseDatabase, typename TModel>
	TestCaseResult ClassifierTestCase<TTestCaseDatabase, TModel>::ProcessResult(const InferenceTestOptions& params)
	{
	auto& output = this->GetOutputs()[0];
	const auto testCaseId = this->GetTestCaseId();

	ClassifierResultProcessor resultProcessor(m_QuantizationParams.first, m_QuantizationParams.second);
	boost::apply_visitor(resultProcessor, output);

	ARMNN_LOG(info) << "= Prediction values for test #" << testCaseId;
	auto it = resultProcessor.GetResultMap().rbegin();
	for (int i=0; i<5 && it != resultProcessor.GetResultMap().rend(); ++i)
	{
	ARMNN_LOG(info) << "Top(" << (i+1) << ") prediction is " << it->second <<
	" with value: " << (it->first);
	++it;
	}

	unsigned int prediction = 0;
	boost::apply_visitor([&](auto&& value)
	{
	prediction = boost::numeric_cast<unsigned int>(
	std::distance(value.begin(), std::max_element(value.begin(), value.end())));
	},
	output);

	// If we're just running the defaultTestCaseIds, each one must be classified correctly.
	if (params.m_IterationCount == 0 && prediction != m_Label)
	{
	ARMNN_LOG(error) << "Prediction for test case " << testCaseId << " (" << prediction << ")" <<
	" is incorrect (should be " << m_Label << ")";
	return TestCaseResult::Failed;
	}

	// If a validation file was provided as input, it checks that the prediction matches.
	if (!m_ValidationPredictions.empty() && prediction != m_ValidationPredictions[testCaseId])
	{
	ARMNN_LOG(error) << "Prediction for test case " << testCaseId << " (" << prediction << ")" <<
	" doesn't match the prediction in the validation file (" << m_ValidationPredictions[testCaseId] << ")";
	return TestCaseResult::Failed;
	}

	// If a validation file was requested as output, it stores the predictions.
	if (m_ValidationPredictionsOut)
	{
	m_ValidationPredictionsOut->push_back(prediction);
	}

	// Updates accuracy stats.
	m_NumInferencesRef++;
	if (prediction == m_Label)
	{
	m_NumCorrectInferencesRef++;
	}

	return TestCaseResult::Ok;
	}

	template <typename TDatabase, typename InferenceModel>
	template <typename TConstructDatabaseCallable, typename TConstructModelCallable>
	ClassifierTestCaseProvider<TDatabase, InferenceModel>::ClassifierTestCaseProvider(
	TConstructDatabaseCallable constructDatabase, TConstructModelCallable constructModel)
	: m_ConstructModel(constructModel)
	, m_ConstructDatabase(constructDatabase)
	, m_NumInferences(0)
	, m_NumCorrectInferences(0)
	{
	}

	template <typename TDatabase, typename InferenceModel>
	void ClassifierTestCaseProvider<TDatabase, InferenceModel>::AddCommandLineOptions(
	boost::program_options::options_description& options)
	{
	namespace po = boost::program_options;

	options.add_options()
	("validation-file-in", po::value<std::string>(&m_ValidationFileIn)->default_value(""),
	"Reads expected predictions from the given file and confirms they match the actual predictions.")
	("validation-file-out", po::value<std::string>(&m_ValidationFileOut)->default_value(""),
	"Predictions are saved to the given file for later use via --validation-file-in.")
	("data-dir,d", po::value<std::string>(&m_DataDir)->required(),
	"Path to directory containing test data");

	InferenceModel::AddCommandLineOptions(options, m_ModelCommandLineOptions);
	}

	template <typename TDatabase, typename InferenceModel>
	bool ClassifierTestCaseProvider<TDatabase, InferenceModel>::ProcessCommandLineOptions(
	const InferenceTestOptions& commonOptions)
	{
	if (!ValidateDirectory(m_DataDir))
	{
	return false;
	}

	ReadPredictions();

	m_Model = m_ConstructModel(commonOptions, m_ModelCommandLineOptions);
	if (!m_Model)
	{
	return false;
	}

	m_Database = std::make_unique<TDatabase>(m_ConstructDatabase(m_DataDir.c_str(), *m_Model));
	if (!m_Database)
	{
	return false;
	}

	return true;
	}

	template <typename TDatabase, typename InferenceModel>
	std::unique_ptr<IInferenceTestCase>
	ClassifierTestCaseProvider<TDatabase, InferenceModel>::GetTestCase(unsigned int testCaseId)
	{
	std::unique_ptr<typename TDatabase::TTestCaseData> testCaseData = m_Database->GetTestCaseData(testCaseId);
	if (testCaseData == nullptr)
	{
	return nullptr;
	}

	return std::make_unique<ClassifierTestCase<TDatabase, InferenceModel>>(
	m_NumInferences,
	m_NumCorrectInferences,
	m_ValidationPredictions,
	m_ValidationFileOut.empty() ? nullptr : &m_ValidationPredictionsOut,
	*m_Model,
	testCaseId,
	testCaseData->m_Label,
	std::move(testCaseData->m_InputImage));
	}

	template <typename TDatabase, typename InferenceModel>
	bool ClassifierTestCaseProvider<TDatabase, InferenceModel>::OnInferenceTestFinished()
	{
	const double accuracy = boost::numeric_cast<double>(m_NumCorrectInferences) /
	boost::numeric_cast<double>(m_NumInferences);
	ARMNN_LOG(info) << std::fixed << std::setprecision(3) << "Overall accuracy: " << accuracy;

	// If a validation file was requested as output, the predictions are saved to it.
	if (!m_ValidationFileOut.empty())
	{
	std::ofstream validationFileOut(m_ValidationFileOut.c_str(), std::ios_base::trunc \| std::ios_base::out);
	if (validationFileOut.good())
	{
	for (const unsigned int prediction : m_ValidationPredictionsOut)
	{
	validationFileOut << prediction << std::endl;
	}
	}
	else
	{
	ARMNN_LOG(error) << "Failed to open output validation file: " << m_ValidationFileOut;
	return false;
	}
	}

	return true;
	}

	template <typename TDatabase, typename InferenceModel>
	void ClassifierTestCaseProvider<TDatabase, InferenceModel>::ReadPredictions()
	{
	// Reads the expected predictions from the input validation file (if provided).
	if (!m_ValidationFileIn.empty())
	{
	std::ifstream validationFileIn(m_ValidationFileIn.c_str(), std::ios_base::in);
	if (validationFileIn.good())
	{
	while (!validationFileIn.eof())
	{
	unsigned int i;
	validationFileIn >> i;
	m_ValidationPredictions.emplace_back(i);
	}
	}
	else
	{
	throw armnn::Exception(boost::str(boost::format("Failed to open input validation file: %1%")
	% m_ValidationFileIn));
	}
	}
	}

	template<typename TConstructTestCaseProvider>
	int InferenceTestMain(int argc,
	char* argv[],
	const std::vector<unsigned int>& defaultTestCaseIds,
	TConstructTestCaseProvider constructTestCaseProvider)
	{
	// Configures logging for both the ARMNN library and this test program.
	#ifdef NDEBUG
	armnn::LogSeverity level = armnn::LogSeverity::Info;
	#else
	armnn::LogSeverity level = armnn::LogSeverity::Debug;
	#endif
	armnn::ConfigureLogging(true, true, level);

	try
	{
	std::unique_ptr<IInferenceTestCaseProvider> testCaseProvider = constructTestCaseProvider();
	if (!testCaseProvider)
	{
	return 1;
	}

	InferenceTestOptions inferenceTestOptions;
	if (!ParseCommandLine(argc, argv, *testCaseProvider, inferenceTestOptions))
	{
	return 1;
	}

	const bool success = InferenceTest(inferenceTestOptions, defaultTestCaseIds, *testCaseProvider);
	return success ? 0 : 1;
	}
	catch (armnn::Exception const& e)
	{
	ARMNN_LOG(fatal) << "Armnn Error: " << e.what();
	return 1;
	}
	}

	//
	// This function allows us to create a classifier inference test based on:
	// - a model file name
	// - which can be a binary or a text file for protobuf formats
	// - an input tensor name
	// - an output tensor name
	// - a set of test case ids
	// - a callback method which creates an object that can return images
	// called 'Database' in these tests
	// - and an input tensor shape
	//
	template<typename TDatabase,
	typename TParser,
	typename TConstructDatabaseCallable>
	int ClassifierInferenceTestMain(int argc,
	char* argv[],
	const char* modelFilename,
	bool isModelBinary,
	const char* inputBindingName,
	const char* outputBindingName,
	const std::vector<unsigned int>& defaultTestCaseIds,
	TConstructDatabaseCallable constructDatabase,
	const armnn::TensorShape* inputTensorShape)

	{
	BOOST_ASSERT(modelFilename);
	BOOST_ASSERT(inputBindingName);
	BOOST_ASSERT(outputBindingName);

	return InferenceTestMain(argc, argv, defaultTestCaseIds,
	[=]
	()
	{
	using InferenceModel = InferenceModel<TParser, typename TDatabase::DataType>;
	using TestCaseProvider = ClassifierTestCaseProvider<TDatabase, InferenceModel>;

	return make_unique<TestCaseProvider>(constructDatabase,
	[&]
	(const InferenceTestOptions &commonOptions,
	typename InferenceModel::CommandLineOptions modelOptions)
	{
	if (!ValidateDirectory(modelOptions.m_ModelDir))
	{
	return std::unique_ptr<InferenceModel>();
	}

	typename InferenceModel::Params modelParams;
	modelParams.m_ModelPath = modelOptions.m_ModelDir + modelFilename;
	modelParams.m_InputBindings = { inputBindingName };
	modelParams.m_OutputBindings = { outputBindingName };

	if (inputTensorShape)
	{
	modelParams.m_InputShapes.push_back(*inputTensorShape);
	}

	modelParams.m_IsModelBinary = isModelBinary;
	modelParams.m_ComputeDevices = modelOptions.GetComputeDevicesAsBackendIds();
	modelParams.m_VisualizePostOptimizationModel = modelOptions.m_VisualizePostOptimizationModel;
	modelParams.m_EnableFp16TurboMode = modelOptions.m_EnableFp16TurboMode;

	return std::make_unique<InferenceModel>(modelParams,
	commonOptions.m_EnableProfiling,
	commonOptions.m_DynamicBackendsPath);
	});
	});
	}

	} // namespace test
	} // namespace armnn