neuralnetworks/1.3/vts/functional/QualityOfServiceTests.cpp - platform/hardware/interfaces - Git at Google

 /*
  * Copyright (C) 2019 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include "1.0/Utils.h"
 #include "1.3/Callbacks.h"
 #include "1.3/Utils.h"
 #include "GeneratedTestHarness.h"
 #include "Utils.h"

 namespace android::hardware::neuralnetworks::V1_3::vts::functional {

 using implementation::ExecutionCallback;
 using implementation::PreparedModelCallback;
 using test_helper::TestBuffer;
 using test_helper::TestModel;
 using V1_1::ExecutionPreference;
 using V1_2::MeasureTiming;
 using V1_2::OutputShape;
 using V1_2::Timing;

 using HidlToken =
         hidl_array<uint8_t, static_cast<uint32_t>(V1_2::Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;

 enum class DeadlineBoundType { NOW, UNLIMITED, SHORT };
 constexpr std::array<DeadlineBoundType, 3> deadlineBounds = {
         DeadlineBoundType::NOW, DeadlineBoundType::UNLIMITED, DeadlineBoundType::SHORT};
 std::string toString(DeadlineBoundType type) {
     switch (type) {
         case DeadlineBoundType::NOW:
             return "NOW";
         case DeadlineBoundType::UNLIMITED:
             return "UNLIMITED";
         case DeadlineBoundType::SHORT:
             return "SHORT";
     }
     LOG(FATAL) << "Unrecognized DeadlineBoundType: " << static_cast<int>(type);
     return {};
 }

 constexpr auto kShortDuration = std::chrono::milliseconds{5};

 using Results = std::tuple<ErrorStatus, hidl_vec<OutputShape>, Timing>;
 using MaybeResults = std::optional<Results>;

 using ExecutionFunction =
         std::function<MaybeResults(const sp<IPreparedModel>& preparedModel, const Request& request,
                                    const OptionalTimePoint& deadline)>;

 static OptionalTimePoint makeDeadline(DeadlineBoundType deadlineBoundType) {
     const auto getNanosecondsSinceEpoch = [](const auto& time) -> uint64_t {
         const auto timeSinceEpoch = time.time_since_epoch();
         return std::chrono::duration_cast<std::chrono::nanoseconds>(timeSinceEpoch).count();
     };

     std::chrono::steady_clock::time_point timePoint;
     switch (deadlineBoundType) {
         case DeadlineBoundType::NOW:
             timePoint = std::chrono::steady_clock::now();
             break;
         case DeadlineBoundType::UNLIMITED:
             timePoint = std::chrono::steady_clock::time_point::max();
             break;
         case DeadlineBoundType::SHORT:
             timePoint = std::chrono::steady_clock::now() + kShortDuration;
             break;
     }

     OptionalTimePoint deadline;
     deadline.nanosecondsSinceEpoch(getNanosecondsSinceEpoch(timePoint));
     return deadline;
 }

 void runPrepareModelTest(const sp<IDevice>& device, const Model& model, Priority priority,
                          std::optional<DeadlineBoundType> deadlineBound) {
     OptionalTimePoint deadline;
     if (deadlineBound.has_value()) {
         deadline = makeDeadline(deadlineBound.value());
     }

     // see if service can handle model
     bool fullySupportsModel = false;
     const Return<void> supportedCall = device->getSupportedOperations_1_3(
             model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
                 ASSERT_EQ(ErrorStatus::NONE, status);
                 ASSERT_NE(0ul, supported.size());
                 fullySupportsModel = std::all_of(supported.begin(), supported.end(),
                                                  [](bool valid) { return valid; });
             });
     ASSERT_TRUE(supportedCall.isOk());

     // launch prepare model
     const sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
     const Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_3(
             model, ExecutionPreference::FAST_SINGLE_ANSWER, priority, deadline,
             hidl_vec<hidl_handle>(), hidl_vec<hidl_handle>(), HidlToken(), preparedModelCallback);
     ASSERT_TRUE(prepareLaunchStatus.isOk());
     ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));

     // retrieve prepared model
     preparedModelCallback->wait();
     const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
     const sp<V1_0::IPreparedModel> preparedModelV1_0 = preparedModelCallback->getPreparedModel();
     const sp<IPreparedModel> preparedModel =
             IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr);

     // The getSupportedOperations_1_3 call returns a list of operations that are
     // guaranteed not to fail if prepareModel_1_3 is called, and
     // 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.
     // If a driver has any doubt that it can prepare an operation, it must
     // return false. So here, if a driver isn't sure if it can support an
     // operation, but reports that it successfully prepared the model, the test
     // can continue.
     if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
         ASSERT_EQ(nullptr, preparedModel.get());
         return;
     }

     // verify return status
     if (!deadlineBound.has_value()) {
         EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
     } else {
         switch (deadlineBound.value()) {
             case DeadlineBoundType::NOW:
             case DeadlineBoundType::SHORT:
                 // Either the driver successfully completed the task or it
                 // aborted and returned MISSED_DEADLINE_*.
                 EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE ||
                             prepareReturnStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT ||
                             prepareReturnStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT);
                 break;
             case DeadlineBoundType::UNLIMITED:
                 // If an unlimited deadline is supplied, we expect the execution to
                 // proceed normally. In this case, check it normally by breaking out
                 // of the switch statement.
                 EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
                 break;
         }
     }
     ASSERT_EQ(prepareReturnStatus == ErrorStatus::NONE, preparedModel.get() != nullptr);
 }

 void runPrepareModelTests(const sp<IDevice>& device, const Model& model) {
     // test priority
     for (auto priority : hidl_enum_range<Priority>{}) {
         SCOPED_TRACE("priority: " + toString(priority));
         if (priority == kDefaultPriority) continue;
         runPrepareModelTest(device, model, priority, {});
     }

     // test deadline
     for (auto deadlineBound : deadlineBounds) {
         SCOPED_TRACE("deadlineBound: " + toString(deadlineBound));
         runPrepareModelTest(device, model, kDefaultPriority, deadlineBound);
     }
 }

 static MaybeResults executeAsynchronously(const sp<IPreparedModel>& preparedModel,
                                           const Request& request,
                                           const OptionalTimePoint& deadline) {
     SCOPED_TRACE("asynchronous");
     const MeasureTiming measure = MeasureTiming::NO;

     // launch execution
     const sp<ExecutionCallback> callback = new ExecutionCallback();
     Return<ErrorStatus> ret = preparedModel->execute_1_3(request, measure, deadline, {}, callback);
     EXPECT_TRUE(ret.isOk());
     EXPECT_EQ(ErrorStatus::NONE, ret.withDefault(ErrorStatus::GENERAL_FAILURE));
     if (!ret.isOk() || ret != ErrorStatus::NONE) return std::nullopt;

     // retrieve execution results
     callback->wait();
     const ErrorStatus status = callback->getStatus();
     hidl_vec<OutputShape> outputShapes = callback->getOutputShapes();
     const Timing timing = callback->getTiming();

     // return results
     return Results{status, std::move(outputShapes), timing};
 }

 static MaybeResults executeSynchronously(const sp<IPreparedModel>& preparedModel,
                                          const Request& request,
                                          const OptionalTimePoint& deadline) {
     SCOPED_TRACE("synchronous");
     const MeasureTiming measure = MeasureTiming::NO;

     // configure results callback
     MaybeResults results;
     const auto cb = [&results](ErrorStatus status, const hidl_vec<OutputShape>& outputShapes,
                                const Timing& timing) {
         results.emplace(status, outputShapes, timing);
     };

     // run execution
     const Return<void> ret =
             preparedModel->executeSynchronously_1_3(request, measure, deadline, {}, cb);
     EXPECT_TRUE(ret.isOk());
     if (!ret.isOk()) return std::nullopt;

     // return results
     return results;
 }

 void runExecutionTest(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
                       const Request& request, const ExecutionContext& context, bool synchronous,
                       DeadlineBoundType deadlineBound) {
     const ExecutionFunction execute = synchronous ? executeSynchronously : executeAsynchronously;
     const auto deadline = makeDeadline(deadlineBound);

     // Perform execution and unpack results.
     const auto results = execute(preparedModel, request, deadline);
     if (!results.has_value()) return;
     const auto& [status, outputShapes, timing] = results.value();

     // Verify no timing information was returned
     EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
     EXPECT_EQ(UINT64_MAX, timing.timeInDriver);

     // Validate deadline information if applicable.
     switch (deadlineBound) {
         case DeadlineBoundType::NOW:
         case DeadlineBoundType::SHORT:
             // Either the driver successfully completed the task or it
             // aborted and returned MISSED_DEADLINE_*.
             ASSERT_TRUE(status == ErrorStatus::NONE ||
                         status == ErrorStatus::MISSED_DEADLINE_TRANSIENT ||
                         status == ErrorStatus::MISSED_DEADLINE_PERSISTENT);
             break;
         case DeadlineBoundType::UNLIMITED:
             // If an unlimited deadline is supplied, we expect the execution to
             // proceed normally. In this case, check it normally by breaking out
             // of the switch statement.
             ASSERT_EQ(ErrorStatus::NONE, status);
             break;
     }

     // If the model output operands are fully specified, outputShapes must be either
     // either empty, or have the same number of elements as the number of outputs.
     ASSERT_TRUE(outputShapes.size() == 0 ||
                 outputShapes.size() == testModel.main.outputIndexes.size());

     // Go through all outputs, check returned output shapes.
     for (uint32_t i = 0; i < outputShapes.size(); i++) {
         EXPECT_TRUE(outputShapes[i].isSufficient);
         const auto& expect = testModel.main.operands[testModel.main.outputIndexes[i]].dimensions;
         const std::vector<uint32_t> actual = outputShapes[i].dimensions;
         EXPECT_EQ(expect, actual);
     }

     // Retrieve execution results.
     ASSERT_TRUE(nn::compliantWithV1_0(request));
     const V1_0::Request request10 = nn::convertToV1_0(request);
     const std::vector<TestBuffer> outputs = context.getOutputBuffers(request10);

     // We want "close-enough" results.
     if (status == ErrorStatus::NONE) {
         checkResults(testModel, outputs);
     }
 }

 void runExecutionTests(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
                        const Request& request, const ExecutionContext& context) {
     for (bool synchronous : {false, true}) {
         for (auto deadlineBound : deadlineBounds) {
             runExecutionTest(preparedModel, testModel, request, context, synchronous,
                              deadlineBound);
         }
     }
 }

 void runTests(const sp<IDevice>& device, const TestModel& testModel) {
     // setup
     const Model model = createModel(testModel);

     // run prepare model tests
     runPrepareModelTests(device, model);

     // prepare model
     sp<IPreparedModel> preparedModel;
     createPreparedModel(device, model, &preparedModel);
     if (preparedModel == nullptr) return;

     // run execution tests
     ExecutionContext context;
     const Request request = nn::convertToV1_3(context.createRequest(testModel));
     runExecutionTests(preparedModel, testModel, request, context);
 }

 class DeadlineTest : public GeneratedTestBase {};

 TEST_P(DeadlineTest, Test) {
     runTests(kDevice, kTestModel);
 }

 INSTANTIATE_GENERATED_TEST(DeadlineTest,
                            [](const TestModel& testModel) { return !testModel.expectFailure; });

 }  // namespace android::hardware::neuralnetworks::V1_3::vts::functional
	/*
	* Copyright (C) 2019 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include "1.0/Utils.h"
	#include "1.3/Callbacks.h"
	#include "1.3/Utils.h"
	#include "GeneratedTestHarness.h"
	#include "Utils.h"

	namespace android::hardware::neuralnetworks::V1_3::vts::functional {

	using implementation::ExecutionCallback;
	using implementation::PreparedModelCallback;
	using test_helper::TestBuffer;
	using test_helper::TestModel;
	using V1_1::ExecutionPreference;
	using V1_2::MeasureTiming;
	using V1_2::OutputShape;
	using V1_2::Timing;

	using HidlToken =
	hidl_array<uint8_t, static_cast<uint32_t>(V1_2::Constant::BYTE_SIZE_OF_CACHE_TOKEN)>;

	enum class DeadlineBoundType { NOW, UNLIMITED, SHORT };
	constexpr std::array<DeadlineBoundType, 3> deadlineBounds = {
	DeadlineBoundType::NOW, DeadlineBoundType::UNLIMITED, DeadlineBoundType::SHORT};
	std::string toString(DeadlineBoundType type) {
	switch (type) {
	case DeadlineBoundType::NOW:
	return "NOW";
	case DeadlineBoundType::UNLIMITED:
	return "UNLIMITED";
	case DeadlineBoundType::SHORT:
	return "SHORT";
	}
	LOG(FATAL) << "Unrecognized DeadlineBoundType: " << static_cast<int>(type);
	return {};
	}

	constexpr auto kShortDuration = std::chrono::milliseconds{5};

	using Results = std::tuple<ErrorStatus, hidl_vec<OutputShape>, Timing>;
	using MaybeResults = std::optional<Results>;

	using ExecutionFunction =
	std::function<MaybeResults(const sp<IPreparedModel>& preparedModel, const Request& request,
	const OptionalTimePoint& deadline)>;

	static OptionalTimePoint makeDeadline(DeadlineBoundType deadlineBoundType) {
	const auto getNanosecondsSinceEpoch = [](const auto& time) -> uint64_t {
	const auto timeSinceEpoch = time.time_since_epoch();
	return std::chrono::duration_cast<std::chrono::nanoseconds>(timeSinceEpoch).count();
	};

	std::chrono::steady_clock::time_point timePoint;
	switch (deadlineBoundType) {
	case DeadlineBoundType::NOW:
	timePoint = std::chrono::steady_clock::now();
	break;
	case DeadlineBoundType::UNLIMITED:
	timePoint = std::chrono::steady_clock::time_point::max();
	break;
	case DeadlineBoundType::SHORT:
	timePoint = std::chrono::steady_clock::now() + kShortDuration;
	break;
	}

	OptionalTimePoint deadline;
	deadline.nanosecondsSinceEpoch(getNanosecondsSinceEpoch(timePoint));
	return deadline;
	}

	void runPrepareModelTest(const sp<IDevice>& device, const Model& model, Priority priority,
	std::optional<DeadlineBoundType> deadlineBound) {
	OptionalTimePoint deadline;
	if (deadlineBound.has_value()) {
	deadline = makeDeadline(deadlineBound.value());
	}

	// see if service can handle model
	bool fullySupportsModel = false;
	const Return<void> supportedCall = device->getSupportedOperations_1_3(
	model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) {
	ASSERT_EQ(ErrorStatus::NONE, status);
	ASSERT_NE(0ul, supported.size());
	fullySupportsModel = std::all_of(supported.begin(), supported.end(),
	[](bool valid) { return valid; });
	});
	ASSERT_TRUE(supportedCall.isOk());

	// launch prepare model
	const sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback();
	const Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_3(
	model, ExecutionPreference::FAST_SINGLE_ANSWER, priority, deadline,
	hidl_vec<hidl_handle>(), hidl_vec<hidl_handle>(), HidlToken(), preparedModelCallback);
	ASSERT_TRUE(prepareLaunchStatus.isOk());
	ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus));

	// retrieve prepared model
	preparedModelCallback->wait();
	const ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus();
	const sp<V1_0::IPreparedModel> preparedModelV1_0 = preparedModelCallback->getPreparedModel();
	const sp<IPreparedModel> preparedModel =
	IPreparedModel::castFrom(preparedModelV1_0).withDefault(nullptr);

	// The getSupportedOperations_1_3 call returns a list of operations that are
	// guaranteed not to fail if prepareModel_1_3 is called, and
	// 'fullySupportsModel' is true i.f.f. the entire model is guaranteed.
	// If a driver has any doubt that it can prepare an operation, it must
	// return false. So here, if a driver isn't sure if it can support an
	// operation, but reports that it successfully prepared the model, the test
	// can continue.
	if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) {
	ASSERT_EQ(nullptr, preparedModel.get());
	return;
	}

	// verify return status
	if (!deadlineBound.has_value()) {
	EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
	} else {
	switch (deadlineBound.value()) {
	case DeadlineBoundType::NOW:
	case DeadlineBoundType::SHORT:
	// Either the driver successfully completed the task or it
	// aborted and returned MISSED_DEADLINE_*.
	EXPECT_TRUE(prepareReturnStatus == ErrorStatus::NONE \|\|
	prepareReturnStatus == ErrorStatus::MISSED_DEADLINE_TRANSIENT \|\|
	prepareReturnStatus == ErrorStatus::MISSED_DEADLINE_PERSISTENT);
	break;
	case DeadlineBoundType::UNLIMITED:
	// If an unlimited deadline is supplied, we expect the execution to
	// proceed normally. In this case, check it normally by breaking out
	// of the switch statement.
	EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus);
	break;
	}
	}
	ASSERT_EQ(prepareReturnStatus == ErrorStatus::NONE, preparedModel.get() != nullptr);
	}

	void runPrepareModelTests(const sp<IDevice>& device, const Model& model) {
	// test priority
	for (auto priority : hidl_enum_range<Priority>{}) {
	SCOPED_TRACE("priority: " + toString(priority));
	if (priority == kDefaultPriority) continue;
	runPrepareModelTest(device, model, priority, {});
	}

	// test deadline
	for (auto deadlineBound : deadlineBounds) {
	SCOPED_TRACE("deadlineBound: " + toString(deadlineBound));
	runPrepareModelTest(device, model, kDefaultPriority, deadlineBound);
	}
	}

	static MaybeResults executeAsynchronously(const sp<IPreparedModel>& preparedModel,
	const Request& request,
	const OptionalTimePoint& deadline) {
	SCOPED_TRACE("asynchronous");
	const MeasureTiming measure = MeasureTiming::NO;

	// launch execution
	const sp<ExecutionCallback> callback = new ExecutionCallback();
	Return<ErrorStatus> ret = preparedModel->execute_1_3(request, measure, deadline, {}, callback);
	EXPECT_TRUE(ret.isOk());
	EXPECT_EQ(ErrorStatus::NONE, ret.withDefault(ErrorStatus::GENERAL_FAILURE));
	if (!ret.isOk() \|\| ret != ErrorStatus::NONE) return std::nullopt;

	// retrieve execution results
	callback->wait();
	const ErrorStatus status = callback->getStatus();
	hidl_vec<OutputShape> outputShapes = callback->getOutputShapes();
	const Timing timing = callback->getTiming();

	// return results
	return Results{status, std::move(outputShapes), timing};
	}

	static MaybeResults executeSynchronously(const sp<IPreparedModel>& preparedModel,
	const Request& request,
	const OptionalTimePoint& deadline) {
	SCOPED_TRACE("synchronous");
	const MeasureTiming measure = MeasureTiming::NO;

	// configure results callback
	MaybeResults results;
	const auto cb = [&results](ErrorStatus status, const hidl_vec<OutputShape>& outputShapes,
	const Timing& timing) {
	results.emplace(status, outputShapes, timing);
	};

	// run execution
	const Return<void> ret =
	preparedModel->executeSynchronously_1_3(request, measure, deadline, {}, cb);
	EXPECT_TRUE(ret.isOk());
	if (!ret.isOk()) return std::nullopt;

	// return results
	return results;
	}

	void runExecutionTest(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
	const Request& request, const ExecutionContext& context, bool synchronous,
	DeadlineBoundType deadlineBound) {
	const ExecutionFunction execute = synchronous ? executeSynchronously : executeAsynchronously;
	const auto deadline = makeDeadline(deadlineBound);

	// Perform execution and unpack results.
	const auto results = execute(preparedModel, request, deadline);
	if (!results.has_value()) return;
	const auto& [status, outputShapes, timing] = results.value();

	// Verify no timing information was returned
	EXPECT_EQ(UINT64_MAX, timing.timeOnDevice);
	EXPECT_EQ(UINT64_MAX, timing.timeInDriver);

	// Validate deadline information if applicable.
	switch (deadlineBound) {
	case DeadlineBoundType::NOW:
	case DeadlineBoundType::SHORT:
	// Either the driver successfully completed the task or it
	// aborted and returned MISSED_DEADLINE_*.
	ASSERT_TRUE(status == ErrorStatus::NONE \|\|
	status == ErrorStatus::MISSED_DEADLINE_TRANSIENT \|\|
	status == ErrorStatus::MISSED_DEADLINE_PERSISTENT);
	break;
	case DeadlineBoundType::UNLIMITED:
	// If an unlimited deadline is supplied, we expect the execution to
	// proceed normally. In this case, check it normally by breaking out
	// of the switch statement.
	ASSERT_EQ(ErrorStatus::NONE, status);
	break;
	}

	// If the model output operands are fully specified, outputShapes must be either
	// either empty, or have the same number of elements as the number of outputs.
	ASSERT_TRUE(outputShapes.size() == 0 \|\|
	outputShapes.size() == testModel.main.outputIndexes.size());

	// Go through all outputs, check returned output shapes.
	for (uint32_t i = 0; i < outputShapes.size(); i++) {
	EXPECT_TRUE(outputShapes[i].isSufficient);
	const auto& expect = testModel.main.operands[testModel.main.outputIndexes[i]].dimensions;
	const std::vector<uint32_t> actual = outputShapes[i].dimensions;
	EXPECT_EQ(expect, actual);
	}

	// Retrieve execution results.
	ASSERT_TRUE(nn::compliantWithV1_0(request));
	const V1_0::Request request10 = nn::convertToV1_0(request);
	const std::vector<TestBuffer> outputs = context.getOutputBuffers(request10);

	// We want "close-enough" results.
	if (status == ErrorStatus::NONE) {
	checkResults(testModel, outputs);
	}
	}

	void runExecutionTests(const sp<IPreparedModel>& preparedModel, const TestModel& testModel,
	const Request& request, const ExecutionContext& context) {
	for (bool synchronous : {false, true}) {
	for (auto deadlineBound : deadlineBounds) {
	runExecutionTest(preparedModel, testModel, request, context, synchronous,
	deadlineBound);
	}
	}
	}

	void runTests(const sp<IDevice>& device, const TestModel& testModel) {
	// setup
	const Model model = createModel(testModel);

	// run prepare model tests
	runPrepareModelTests(device, model);

	// prepare model
	sp<IPreparedModel> preparedModel;
	createPreparedModel(device, model, &preparedModel);
	if (preparedModel == nullptr) return;

	// run execution tests
	ExecutionContext context;
	const Request request = nn::convertToV1_3(context.createRequest(testModel));
	runExecutionTests(preparedModel, testModel, request, context);
	}

	class DeadlineTest : public GeneratedTestBase {};

	TEST_P(DeadlineTest, Test) {
	runTests(kDevice, kTestModel);
	}

	INSTANTIATE_GENERATED_TEST(DeadlineTest,
	[](const TestModel& testModel) { return !testModel.expectFailure; });

	} // namespace android::hardware::neuralnetworks::V1_3::vts::functional