| /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. |
| |
| Licensed under the Apache License, Version 2.0 (the "License"); |
| you may not use this file except in compliance with the License. |
| You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| ==============================================================================*/ |
| #include <cstdarg> |
| #include <cstdio> |
| #include <cstdlib> |
| #include <fstream> |
| #include <map> |
| #include <sstream> |
| #include <tuple> |
| #include <utility> |
| |
| #include <gtest/gtest.h> |
| #include "re2/re2.h" |
| #include "tensorflow/core/platform/env.h" |
| #include "tensorflow/core/platform/subprocess.h" |
| #include "tensorflow/core/util/command_line_flags.h" |
| #include "tensorflow/lite/testing/parse_testdata.h" |
| #include "tensorflow/lite/testing/tflite_driver.h" |
| #include "tensorflow/lite/testing/util.h" |
| |
| namespace tflite { |
| namespace testing { |
| |
| namespace { |
| bool FLAGS_ignore_known_bugs = true; |
| // As archive file names are test-specific, no default is possible. |
| // |
| // This test supports input as both zip and tar, as a stock android image does |
| // not have unzip but does have tar. |
| string* FLAGS_zip_file_path = new string; |
| string* FLAGS_tar_file_path = new string; |
| #ifndef __ANDROID__ |
| string* FLAGS_unzip_binary_path = new string("/usr/bin/unzip"); |
| string* FLAGS_tar_binary_path = new string("/bin/tar"); |
| #else |
| string* FLAGS_unzip_binary_path = new string("/system/bin/unzip"); |
| string* FLAGS_tar_binary_path = new string("/system/bin/tar"); |
| #endif |
| bool FLAGS_use_nnapi = false; |
| bool FLAGS_ignore_unsupported_nnapi = false; |
| } // namespace |
| |
| // TensorFlow system environment for file system called. |
| tensorflow::Env* env = tensorflow::Env::Default(); |
| |
| // Already known broken tests. |
| // Key is a substring of the test name and value is pair of a bug number and |
| // whether this test should be always ignored regardless of `ignore_known_bugs`. |
| // if `always_ignore` is false, tests are expected to fail when |
| // --test_arg=--ignore_known_bugs=false |
| using BrokenTestMap = |
| std::map</* test_name */ string, |
| std::pair</* bug_number */ string, /* always_ignore */ bool>>; |
| // TODO(ahentz): make sure we clean this list up frequently. |
| const BrokenTestMap& GetKnownBrokenTests() { |
| static const BrokenTestMap* const kBrokenTests = new BrokenTestMap({ |
| // Select kernel doesn't support broadcasting yet. |
| {R"(^\/where.*1,2,3,1)", {"134692786", false}}, |
| |
| // TODO(b/194364155): TF and TFLite have different behaviors when output |
| // nan values in LocalResponseNorm ops. |
| {R"(^\/local_response_norm.*alpha=-3.*beta=2)", {"194364155", true}}, |
| {R"(^\/local_response_norm.*alpha=(None|2).*beta=2.*bias=-0\.1.*depth_radius=(0|1).*input_shape=\[3,15,14,3\])", |
| {"194364155", true}}, |
| }); |
| return *kBrokenTests; |
| } |
| |
| // Additional list of tests that are expected to fail when |
| // --test_arg=--ignore_known_bugs=false |
| // and |
| // --test_arg=--use_nnapi=true |
| // Note that issues related to lack of NNAPI support for a particular op are |
| // handled separately; this list is specifically for broken cases where |
| // execution produces broken output. |
| // Key is a substring of the test name and value is a bug number. |
| const std::map<string, string>& GetKnownBrokenNnapiTests() { |
| static const std::map<string, string>* const kBrokenNnapiTests = |
| new std::map<string, string>({ |
| // Certain NNAPI kernels silently fail with int32 types. |
| {R"(^\/add.*dtype=tf\.int32)", "122987564"}, |
| {R"(^\/concat.*dtype=tf\.int32)", "122987564"}, |
| {R"(^\/mul.*dtype=tf\.int32)", "122987564"}, |
| {R"(^\/space_to_depth.*dtype=tf\.int32)", "122987564"}, |
| |
| // Certain NNAPI fully_connected shape permutations fail. |
| {R"(^\/fully_connected_constant_filter=True.*shape1=\[3,3\])", |
| "122987564"}, |
| {R"(^\/fully_connected_constant_filter=True.*shape1=\[4,4\])", |
| "122987564"}, |
| {R"(^\/fully_connected.*shape1=\[3,3\].*transpose_b=True)", |
| "122987564"}, |
| {R"(^\/fully_connected.*shape1=\[4,4\].*shape2=\[4,1\])", |
| "122987564"}, |
| }); |
| return *kBrokenNnapiTests; |
| } |
| |
| // List of quantize tests that are probably to fail. |
| // Quantized tflite models has high diff error with tensorflow models. |
| // Key is a substring of the test name and value is a bug number. |
| // TODO(b/134594898): Remove these bugs and corresponding codes or move them to |
| // kBrokenTests after b/134594898 is fixed. |
| const std::map<string, string>& GetKnownQuantizeBrokenTests() { |
| static const std::map<string, string>* const kQuantizeBrokenTests = |
| new std::map<string, string>({ |
| {R"(^\/conv.*fully_quantize=True)", "134594898"}, |
| {R"(^\/depthwiseconv.*fully_quantize=True)", "134594898"}, |
| {R"(^\/sum.*fully_quantize=True)", "134594898"}, |
| {R"(^\/l2norm.*fully_quantize=True)", "134594898"}, |
| {R"(^\/prelu.*fully_quantize=True)", "156112683"}, |
| }); |
| return *kQuantizeBrokenTests; |
| } |
| |
| // Allows test data to be unarchived into a temporary directory and makes |
| // sure those temporary directories are removed later. |
| class ArchiveEnvironment : public ::testing::Environment { |
| public: |
| ~ArchiveEnvironment() override {} |
| |
| // Delete all temporary directories on teardown. |
| void TearDown() override { |
| for (const auto& dir : temporary_directories_) { |
| int64_t undeleted_dirs, undeleted_files; |
| TF_CHECK_OK( |
| env->DeleteRecursively(dir, &undeleted_dirs, &undeleted_files)); |
| } |
| temporary_directories_.clear(); |
| } |
| |
| // Unarchive `archive` file into a new temporary directory `out_dir`. |
| tensorflow::Status UnArchive(const string& zip, const string& tar, |
| string* out_dir) { |
| string dir; |
| TF_CHECK_OK(MakeTemporaryDirectory(&dir)); |
| tensorflow::SubProcess proc; |
| if (!zip.empty()) { |
| string unzip_binary = *FLAGS_unzip_binary_path; |
| TF_CHECK_OK(env->FileExists(unzip_binary)); |
| TF_CHECK_OK(env->FileExists(zip)); |
| proc.SetProgram(unzip_binary, {"unzip", "-d", dir, zip}); |
| } else { |
| string tar_binary = *FLAGS_tar_binary_path; |
| TF_CHECK_OK(env->FileExists(tar_binary)); |
| TF_CHECK_OK(env->FileExists(tar)); |
| // 'o' needs to be explicitly set on Android so that |
| // untarring works as non-root (otherwise tries to chown |
| // files, which fails) |
| proc.SetProgram(tar_binary, {"tar", "xfo", tar, "-C", dir}); |
| } |
| proc.SetChannelAction(tensorflow::CHAN_STDOUT, tensorflow::ACTION_PIPE); |
| proc.SetChannelAction(tensorflow::CHAN_STDERR, tensorflow::ACTION_PIPE); |
| if (!proc.Start()) |
| return tensorflow::Status(tensorflow::error::UNKNOWN, |
| "unzip couldn't start"); |
| string out, err; |
| int status = proc.Communicate(nullptr, &out, &err); |
| if (WEXITSTATUS(status) == 0) { |
| *out_dir = dir; |
| return tensorflow::Status::OK(); |
| } else { |
| return tensorflow::Status(tensorflow::error::UNKNOWN, |
| "unzip failed. " |
| "stdout:\n" + |
| out + "\nstderr:\n" + err); |
| } |
| } |
| |
| private: |
| // Make a temporary directory and return its name in `temporary`. |
| tensorflow::Status MakeTemporaryDirectory(string* temporary) { |
| if (env->LocalTempFilename(temporary)) { |
| TF_CHECK_OK(env->CreateDir(*temporary)); |
| temporary_directories_.push_back(*temporary); |
| return tensorflow::Status::OK(); |
| } |
| return tensorflow::Status(tensorflow::error::UNKNOWN, |
| "make temporary directory failed"); |
| } |
| |
| std::vector<string> temporary_directories_; |
| }; |
| |
| // Return the singleton archive_environment. |
| ArchiveEnvironment* archive_environment() { |
| static ArchiveEnvironment* env = new ArchiveEnvironment; |
| return env; |
| } |
| |
| // Read the manifest.txt out of the unarchived archive file. Specifically |
| // `original_file` is the original zip file for error messages. `dir` is |
| // the temporary directory where the archive file has been unarchived and |
| // `test_paths` is the list of test prefixes that were in the manifest. |
| // Note, it is an error for a manifest to contain no tests. |
| tensorflow::Status ReadManifest(const string& original_file, const string& dir, |
| std::vector<string>* test_paths) { |
| // Read the newline delimited list of entries in the manifest. |
| std::ifstream manifest_fp(dir + "/manifest.txt"); |
| string manifest((std::istreambuf_iterator<char>(manifest_fp)), |
| std::istreambuf_iterator<char>()); |
| size_t pos = 0; |
| int added = 0; |
| while (true) { |
| size_t end_pos = manifest.find('\n', pos); |
| if (end_pos == string::npos) break; |
| string filename = manifest.substr(pos, end_pos - pos); |
| test_paths->push_back(dir + "/" + filename); |
| pos = end_pos + 1; |
| added += 1; |
| } |
| if (!added) { |
| string message = "Test had no examples: " + original_file; |
| return tensorflow::Status(tensorflow::error::UNKNOWN, message); |
| } |
| return tensorflow::Status::OK(); |
| } |
| |
| // Get a list of tests from either zip or tar file |
| std::vector<string> UnarchiveAndFindTestNames(const string& zip_file, |
| const string& tar_file) { |
| if (zip_file.empty() && tar_file.empty()) { |
| TF_CHECK_OK(tensorflow::Status(tensorflow::error::UNKNOWN, |
| "Neither zip_file nor tar_file was given")); |
| } |
| string decompress_tmp_dir; |
| TF_CHECK_OK(archive_environment()->UnArchive(zip_file, tar_file, |
| &decompress_tmp_dir)); |
| std::vector<string> stuff; |
| if (!zip_file.empty()) { |
| TF_CHECK_OK(ReadManifest(zip_file, decompress_tmp_dir, &stuff)); |
| } else { |
| TF_CHECK_OK(ReadManifest(tar_file, decompress_tmp_dir, &stuff)); |
| } |
| return stuff; |
| } |
| |
| class OpsTest : public ::testing::TestWithParam<string> {}; |
| |
| TEST_P(OpsTest, RunZipTests) { |
| string test_path_and_label = GetParam(); |
| string test_path = test_path_and_label; |
| string label = test_path_and_label; |
| size_t end_pos = test_path_and_label.find(' '); |
| if (end_pos != string::npos) { |
| test_path = test_path_and_label.substr(0, end_pos); |
| label = test_path_and_label.substr(end_pos + 1); |
| } |
| string tflite_test_case = test_path + "_tests.txt"; |
| string tflite_dir = test_path.substr(0, test_path.find_last_of('/')); |
| string test_name = label.substr(label.find_last_of('/')); |
| |
| std::ifstream tflite_stream(tflite_test_case); |
| ASSERT_TRUE(tflite_stream.is_open()) << tflite_test_case; |
| tflite::testing::TfLiteDriver test_driver( |
| FLAGS_use_nnapi ? TfLiteDriver::DelegateType::kNnapi |
| : TfLiteDriver::DelegateType::kNone); |
| bool fully_quantize = false; |
| if (label.find("fully_quantize=True") != std::string::npos) { |
| // TODO(b/134594898): Tighten this constraint. |
| test_driver.SetThreshold(0.2, 0.1); |
| fully_quantize = true; |
| } |
| |
| test_driver.SetModelBaseDir(tflite_dir); |
| |
| auto broken_tests = GetKnownBrokenTests(); |
| if (FLAGS_use_nnapi) { |
| for (const auto& t : GetKnownBrokenNnapiTests()) { |
| broken_tests[t.first] = |
| std::make_pair(t.second, /* always ignore */ false); |
| } |
| } |
| auto quantize_broken_tests = GetKnownQuantizeBrokenTests(); |
| |
| bool result = tflite::testing::ParseAndRunTests(&tflite_stream, &test_driver); |
| string message = test_driver.GetErrorMessage(); |
| |
| if (!fully_quantize) { |
| string bug_number; |
| bool always_ignore; |
| for (const auto& p : broken_tests) { |
| if (RE2::PartialMatch(test_name, p.first)) { |
| std::tie(bug_number, always_ignore) = p.second; |
| break; |
| } |
| } |
| if (bug_number.empty()) { |
| if (FLAGS_use_nnapi && FLAGS_ignore_unsupported_nnapi && !result) { |
| EXPECT_EQ(message, string("Failed to invoke interpreter")) << message; |
| } else { |
| EXPECT_TRUE(result) << message; |
| } |
| } else { |
| if (FLAGS_ignore_known_bugs) { |
| EXPECT_FALSE(result) << "Test was expected to fail but is now passing; " |
| "you can mark http://b/" |
| << bug_number << " as fixed! Yay!"; |
| } else { |
| EXPECT_TRUE(result || always_ignore) |
| << message << ": Possibly due to http://b/" << bug_number; |
| } |
| } |
| } else { |
| if (!result) { |
| string bug_number; |
| // See if the tests are potential quantize failures. |
| for (const auto& p : quantize_broken_tests) { |
| if (RE2::PartialMatch(test_name, p.first)) { |
| bug_number = p.second; |
| break; |
| } |
| } |
| EXPECT_FALSE(bug_number.empty()); |
| } |
| } |
| } |
| |
| struct ZipPathParamName { |
| template <class ParamType> |
| string operator()(const ::testing::TestParamInfo<ParamType>& info) const { |
| string param_name = info.param; |
| size_t last_slash = param_name.find_last_of("\\/"); |
| if (last_slash != string::npos) { |
| param_name = param_name.substr(last_slash); |
| } |
| for (size_t index = 0; index < param_name.size(); ++index) { |
| if (!isalnum(param_name[index]) && param_name[index] != '_') |
| param_name[index] = '_'; |
| } |
| return param_name; |
| } |
| }; |
| |
| INSTANTIATE_TEST_CASE_P(tests, OpsTest, |
| ::testing::ValuesIn(UnarchiveAndFindTestNames( |
| *FLAGS_zip_file_path, *FLAGS_tar_file_path)), |
| ZipPathParamName()); |
| |
| } // namespace testing |
| } // namespace tflite |
| |
| int main(int argc, char** argv) { |
| ::testing::AddGlobalTestEnvironment(tflite::testing::archive_environment()); |
| |
| std::vector<tensorflow::Flag> flags = { |
| tensorflow::Flag( |
| "ignore_known_bugs", &tflite::testing::FLAGS_ignore_known_bugs, |
| "If a particular model is affected by a known bug, the " |
| "corresponding test should expect the outputs to not match."), |
| tensorflow::Flag( |
| "tar_file_path", tflite::testing::FLAGS_tar_file_path, |
| "Required (or zip_file_path): Location of the test tar file."), |
| tensorflow::Flag( |
| "zip_file_path", tflite::testing::FLAGS_zip_file_path, |
| "Required (or tar_file_path): Location of the test zip file."), |
| tensorflow::Flag("unzip_binary_path", |
| tflite::testing::FLAGS_unzip_binary_path, |
| "Location of a suitable unzip binary."), |
| tensorflow::Flag("tar_binary_path", |
| tflite::testing::FLAGS_tar_binary_path, |
| "Location of a suitable tar binary."), |
| tensorflow::Flag("use_nnapi", &tflite::testing::FLAGS_use_nnapi, |
| "Whether to enable the NNAPI delegate"), |
| tensorflow::Flag("ignore_unsupported_nnapi", |
| &tflite::testing::FLAGS_ignore_unsupported_nnapi, |
| "Don't fail tests just because delegation to NNAPI " |
| "is not possible")}; |
| bool success = tensorflow::Flags::Parse(&argc, argv, flags); |
| if (!success || (argc == 2 && !strcmp(argv[1], "--helpfull"))) { |
| fprintf(stderr, "%s", tensorflow::Flags::Usage(argv[0], flags).c_str()); |
| return 1; |
| } |
| |
| if (!tflite::testing::TfLiteDriver::InitTestDelegateProviders( |
| &argc, const_cast<const char**>(argv))) { |
| return EXIT_FAILURE; |
| } |
| |
| ::tflite::LogToStderr(); |
| // TODO(mikie): googletest arguments do not work - maybe the tensorflow flags |
| // parser removes them? |
| ::testing::InitGoogleTest(&argc, argv); |
| return RUN_ALL_TESTS(); |
| } |