[1/2] Remove caffe2 db and distributed from build system (#125092)
This PR tries to decompose https://github.com/pytorch/pytorch/pull/122527 into a smaller one. Caffe2 db, distributed and some binaries have been removed.
To be noted, this was inspired and is co-dev with @r-barnes.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/125092
Approved by: https://github.com/malfet
diff --git a/.ci/docker/common/install_base.sh b/.ci/docker/common/install_base.sh
index c2c3003..ebaa178 100755
--- a/.ci/docker/common/install_base.sh
+++ b/.ci/docker/common/install_base.sh
@@ -113,7 +113,6 @@
glibc-devel \
glibc-headers \
glog-devel \
- hiredis-devel \
libstdc++-devel \
libsndfile-devel \
make \
diff --git a/.ci/docker/common/install_db.sh b/.ci/docker/common/install_db.sh
index 7e18947..7e72340 100755
--- a/.ci/docker/common/install_db.sh
+++ b/.ci/docker/common/install_db.sh
@@ -4,11 +4,6 @@
install_ubuntu() {
apt-get update
- apt-get install -y --no-install-recommends \
- libhiredis-dev \
- libleveldb-dev \
- liblmdb-dev \
- libsnappy-dev
# Cleanup
apt-get autoclean && apt-get clean
@@ -20,12 +15,6 @@
# See http://fedoraproject.org/wiki/EPEL
yum --enablerepo=extras install -y epel-release
- yum install -y \
- hiredis-devel \
- leveldb-devel \
- lmdb-devel \
- snappy-devel
-
# Cleanup
yum clean all
rm -rf /var/cache/yum
diff --git a/BUILD.bazel b/BUILD.bazel
index ba07c91..a48f4b0 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -504,16 +504,6 @@
)
filegroup(
- name = "caffe2_distributed_srcs",
- srcs = [
- "caffe2/distributed/file_store_handler.cc",
- "caffe2/distributed/file_store_handler_op.cc",
- "caffe2/distributed/store_handler.cc",
- "caffe2/distributed/store_ops.cc",
- ],
-)
-
-filegroup(
name = "caffe2_ideep_srcs",
srcs = [
"caffe2/ideep/operators/adam_op.cc",
@@ -1011,8 +1001,6 @@
"caffe2/core/common_cudnn.cc",
"caffe2/core/common_gpu.cc",
"caffe2/core/event_gpu.cc",
- "caffe2/db/create_db_op_gpu.cc",
- "caffe2/distributed/file_store_handler_op_gpu.cc",
"caffe2/operators/communicator_op_gpu.cc",
"caffe2/operators/concat_split_op_gpu.cc",
"caffe2/operators/conv_op_cache_cudnn.cc",
@@ -1262,8 +1250,6 @@
"caffe2/core/nomnigraph/include/nomnigraph/Support/*.h",
"caffe2/core/nomnigraph/include/nomnigraph/Transformations/*.h",
"caffe2/core/nomnigraph/tests/*.h",
- "caffe2/db/*.h",
- "caffe2/distributed/*.h",
"caffe2/ideep/*.h",
"caffe2/ideep/operators/*.h",
"caffe2/ideep/operators/quantization/*.h",
@@ -1335,11 +1321,8 @@
cc_library(
name = "caffe2",
srcs = [
- "caffe2/db/create_db_op.cc",
- "caffe2/db/protodb.cc",
"caffe2/share/contrib/depthwise/depthwise3x3_conv_op.cc",
":caffe2_core_srcs",
- ":caffe2_distributed_srcs",
":caffe2_ideep_srcs",
":caffe2_onnx_srcs",
":caffe2_operators_srcs",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9ce1e06..d61f3a76 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -230,9 +230,7 @@
option(USE_FAKELOWP "Use FakeLowp operators" OFF)
option(USE_GFLAGS "Use GFLAGS" OFF)
option(USE_GLOG "Use GLOG" OFF)
-option(USE_LEVELDB "Use LEVELDB" OFF)
option(USE_LITE_PROTO "Use lite protobuf instead of full." OFF)
-option(USE_LMDB "Use LMDB" OFF)
option(USE_MAGMA "Use MAGMA" ON)
option(USE_METAL "Use Metal for Caffe2 iOS build" ON)
option(USE_PYTORCH_METAL "Use Metal for PyTorch iOS build" OFF)
@@ -269,8 +267,6 @@
option(USE_PROF "Use profiling" OFF)
option(USE_QNNPACK "Use QNNPACK (quantized 8-bit operators)" ON)
option(USE_PYTORCH_QNNPACK "Use ATen/QNNPACK (quantized 8-bit operators)" ON)
-option(USE_REDIS "Use Redis" OFF)
-option(USE_ROCKSDB "Use RocksDB" OFF)
option(USE_SNPE "Use Qualcomm's SNPE library" OFF)
option(USE_SYSTEM_EIGEN_INSTALL
"Use system Eigen instead of the one under third_party" OFF)
@@ -292,7 +288,6 @@
option(USE_VULKAN_RELAXED_PRECISION "Vulkan - Use relaxed precision math in the kernels (mediump)" OFF)
# option USE_XNNPACK: try to enable xnnpack by default.
option(USE_XNNPACK "Use XNNPACK" ON)
-option(USE_ZMQ "Use ZMQ" OFF)
option(USE_ZSTD "Use ZSTD" OFF)
option(USE_ROCM_KERNEL_ASSERT "Use Kernel Assert for ROCm" OFF)
# Ensure that an ITT build is the default for x86 CPUs
diff --git a/binaries/CMakeLists.txt b/binaries/CMakeLists.txt
index 15f47bf..b728cd1 100644
--- a/binaries/CMakeLists.txt
+++ b/binaries/CMakeLists.txt
@@ -8,20 +8,6 @@
endif()
if(BUILD_CAFFE2)
- caffe2_binary_target("convert_caffe_image_db.cc")
- caffe2_binary_target("convert_db.cc")
- caffe2_binary_target("make_cifar_db.cc")
- caffe2_binary_target("make_mnist_db.cc")
- caffe2_binary_target("predictor_verifier.cc")
- caffe2_binary_target("speed_benchmark.cc")
- caffe2_binary_target("split_db.cc")
- caffe2_binary_target("print_registered_core_operators.cc")
-
- if(USE_OBSERVERS)
- caffe2_binary_target(caffe2_benchmark "caffe2_benchmark.cc" "benchmark_helper.cc")
- target_include_directories(caffe2_benchmark PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../modules)
- endif()
-
caffe2_binary_target("at_launch_benchmark.cc")
target_include_directories(at_launch_benchmark PUBLIC
${CMAKE_BINARY_DIR}/aten/src)
@@ -29,12 +15,6 @@
caffe2_binary_target("intra_inter_benchmark.cc")
target_include_directories(intra_inter_benchmark PUBLIC
${CMAKE_BINARY_DIR}/aten/src)
-
- caffe2_binary_target("run_plan.cc")
- caffe2_binary_target("db_throughput.cc")
-
- # ---[ tutorials
- caffe2_binary_target("tutorial_blob.cc")
endif()
caffe2_binary_target("parallel_info.cc")
@@ -77,34 +57,11 @@
endif()
endif()
-if(USE_ZMQ)
- caffe2_binary_target("zmq_feeder.cc")
- target_link_libraries(zmq_feeder ${ZMQ_LIBRARIES})
-endif()
-
if(USE_MPI)
caffe2_binary_target("run_plan_mpi.cc")
target_link_libraries(run_plan_mpi ${MPI_CXX_LIBRARIES})
endif()
-if(USE_OPENCV AND USE_LEVELDB)
- caffe2_binary_target("convert_encoded_to_raw_leveldb.cc")
- target_link_libraries(
- convert_encoded_to_raw_leveldb
- ${OpenCV_LIBS} ${LevelDB_LIBRARIES} ${Snappy_LIBRARIES})
-endif()
-
-if(USE_OPENCV)
- caffe2_binary_target("make_image_db.cc")
- target_link_libraries(make_image_db ${OpenCV_LIBS})
- caffe2_binary_target("convert_image_to_tensor.cc")
- target_link_libraries(convert_image_to_tensor ${OpenCV_LIBS})
-endif()
-
-if(USE_OBSERVERS AND USE_OPENCV)
- caffe2_binary_target("convert_and_benchmark.cc")
- target_link_libraries(convert_and_benchmark ${OpenCV_LIBS})
-endif()
caffe2_binary_target("dump_operator_names.cc")
caffe2_binary_target("optimize_for_mobile.cc")
diff --git a/binaries/benchmark_helper.cc b/binaries/benchmark_helper.cc
deleted file mode 100644
index 117e118..0000000
--- a/binaries/benchmark_helper.cc
+++ /dev/null
@@ -1,523 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <chrono>
-#include <fstream>
-#include <iostream>
-#include <string>
-#include <thread>
-#ifdef _WIN32
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif
-#include <windows.h>
-#include <psapi.h>
-#endif
-
-#include <binaries/benchmark_helper.h>
-#include "caffe2/core/blob_serialization.h"
-#ifdef __CUDA_ARCH__
-#include "caffe2/core/context_gpu.h"
-#endif
-#include "caffe2/core/init.h"
-#include "caffe2/core/logging.h"
-#include "caffe2/core/net.h"
-#include "caffe2/core/operator.h"
-#include "caffe2/core/tensor_int8.h"
-#include "caffe2/utils/bench_utils.h"
-#include "caffe2/utils/string_utils.h"
-#include <observers/net_observer_reporter_print.h>
-#include <observers/observer_config.h>
-#include <observers/perf_observer.h>
-
-#if defined(TARGET_OS_MAC) || \
-defined(TARGET_OS_IPHONE) || \
-defined(TARGET_IPHONE_SIMULATOR)
-#include <malloc/malloc.h>
-#else
-#include <malloc.h>
-#endif
-
-
-void observerConfig() {
- caffe2::ClearGlobalNetObservers();
- caffe2::AddGlobalNetObserverCreator([](caffe2::NetBase* subject) {
- return std::make_unique<caffe2::PerfNetObserver>(subject);
- });
- caffe2::ObserverConfig::setReporter(
- std::make_unique<caffe2::NetObserverReporterPrint>());
-}
-
-bool backendCudaSet(const string& backend) {
- bool run_on_gpu = false;
- if (backend == "cuda") {
-#ifdef __CUDA_ARCH__
- if (caffe2::HasCudaGPU()) {
- run_on_gpu = true;
- } else {
- CAFFE_THROW("NO GPU support on this host machine");
- }
-#else
- CAFFE_THROW("NO GPU support");
-#endif
- }
- return run_on_gpu;
-}
-
-void setDeviceType(caffe2::NetDef* net_def, caffe2::DeviceType& run_dev) {
- for (int j = 0; j < net_def->op_size(); j++) {
- caffe2::OperatorDef* op = net_def->mutable_op(j);
- op->mutable_device_option()->set_device_type(caffe2::TypeToProto(run_dev));
- }
-}
-
-void setOperatorEngine(caffe2::NetDef* net_def, const string& backend) {
- if (backend != "builtin") {
- string engine = backend == "nnpack"
- ? "NNPACK"
- : backend == "eigen" ? "EIGEN"
- : backend == "mkl" ? "MKLDNN"
- : backend == "cuda"
- ? "CUDA"
- : backend == "dnnlowp" ? "DNNLOWP"
- : backend == "dnnlowp_acc16"
- ? "DNNLOWP_ACC16"
- : backend == "default" ? "" : "NONE";
- CAFFE_ENFORCE(engine != "NONE", "Backend is not supported");
- for (int i = 0; i < net_def->op_size(); i++) {
- caffe2::OperatorDef* op_def = net_def->mutable_op(i);
- op_def->set_engine(engine);
- }
- }
-}
-
-int loadInput(
- shared_ptr<caffe2::Workspace> workspace,
- const bool run_on_gpu,
- map<string, caffe2::TensorProtos>& tensor_protos_map,
- const string& input,
- const string& input_file,
- const string& input_dims,
- const string& input_type) {
- // How many input blobs are in the inputs
- int blob_num = 1;
- // Load input.
- if (input.size()) {
- vector<string> input_names = caffe2::split(',', input);
- if (input_file.size()) {
- vector<string> input_files = caffe2::split(',', input_file);
- CAFFE_ENFORCE_EQ(
- input_names.size(),
- input_files.size(),
- "Input name and file should have the same number.");
- for (int i = 0; i < input_names.size(); ++i) {
- caffe2::TensorProtos tensor_protos;
- CAFFE_ENFORCE(
- caffe2::ReadProtoFromFile(input_files[i], &tensor_protos));
- workspace->CreateBlob(input_names[i]);
- tensor_protos_map.insert(std::make_pair(input_names[i], tensor_protos));
- }
- // Check that all blobs have the same number of entries
- blob_num = tensor_protos_map[input_names[0]].protos_size();
- for (int i = 1; i < input_names.size(); ++i) {
- int bnum = tensor_protos_map[input_names[i]].protos_size();
- CAFFE_ENFORCE_EQ(
- blob_num,
- bnum,
- "Number of blobs are not the same for all inputs");
- }
- } else if (input_dims.size() || input_type.size()) {
- CAFFE_ENFORCE_GE(
- input_dims.size(),
- 0,
- "Input dims must be specified when input tensors are used.");
- CAFFE_ENFORCE_GE(
- input_type.size(),
- 0,
- "Input type must be specified when input tensors are used.");
-
- vector<string> input_dims_list = caffe2::split(';', input_dims);
- CAFFE_ENFORCE_EQ(
- input_names.size(),
- input_dims_list.size(),
- "Input name and dims should have the same number of items.");
- vector<string> input_type_list = caffe2::split(';', input_type);
- CAFFE_ENFORCE_EQ(
- input_names.size(),
- input_type_list.size(),
- "Input name and type should have the same number of items.");
- for (size_t i = 0; i < input_names.size(); ++i) {
- vector<string> input_dims_str = caffe2::split(',', input_dims_list[i]);
- vector<int> input_dims;
- for (const string& s : input_dims_str) {
- input_dims.push_back(std::stoi(s));
- }
- caffe2::Blob* blob = workspace->GetBlob(input_names[i]);
- if (blob == nullptr) {
- blob = workspace->CreateBlob(input_names[i]);
- }
- if (run_on_gpu) {
- LOG(INFO) << "Running on GPU.";
-#ifdef __CUDA_ARCH__
- caffe2::TensorCUDA* tensor = blob->GetMutable<caffe2::TensorCUDA>();
- TORCH_CHECK_NOTNULL(tensor);
- tensor->Resize(input_dims);
- if (input_type_list[i] == "uint8_t") {
- tensor->mutable_data<uint8_t>();
- } else if (input_type_list[i] == "float") {
- tensor->mutable_data<float>();
- } else {
- CAFFE_THROW("Unsupported input type: ", input_type_list[i]);
- }
-#else
- CAFFE_THROW("Not support GPU on mobile.");
-#endif
- } else {
- if (input_type_list[i] == "uint8_t") {
- caffe2::int8::Int8TensorCPU* tensor =
- blob->GetMutable<caffe2::int8::Int8TensorCPU>();
- TORCH_CHECK_NOTNULL(tensor);
- tensor->t.Resize(input_dims);
- tensor->t.mutable_data<uint8_t>();
- } else if (input_type_list[i] == "float") {
- caffe2::TensorCPU* tensor = BlobGetMutableTensor(blob, caffe2::CPU);
- TORCH_CHECK_NOTNULL(tensor);
- tensor->Resize(input_dims);
- tensor->mutable_data<float>();
- } else if (input_type_list[i] == "int") {
- caffe2::TensorCPU* tensor = BlobGetMutableTensor(blob, caffe2::CPU);
- TORCH_CHECK_NOTNULL(tensor);
- tensor->Resize(input_dims);
- tensor->mutable_data<int>();
- } else {
- CAFFE_THROW("Unsupported input type: ", input_type_list[i]);
- }
- }
- }
- } else {
- CAFFE_THROW(
- "You requested input tensors, but neither input_file nor "
- "input_dims is set.");
- }
- }
- return blob_num;
-}
-
-void fillInputBlob(
- shared_ptr<caffe2::Workspace> workspace,
- map<string, caffe2::TensorProtos>& tensor_protos_map,
- int iteration) {
- if (tensor_protos_map.empty()) {
- return;
- }
- static caffe2::TensorDeserializer deserializer;
- for (auto& tensor_kv : tensor_protos_map) {
- caffe2::Blob* blob = workspace->GetBlob(tensor_kv.first);
- if (blob == nullptr) {
- blob = workspace->CreateBlob(tensor_kv.first);
- }
- // todo: support gpu and make this function a template
- int protos_size = tensor_kv.second.protos_size();
- if (protos_size == 1 && iteration > 0) {
- // Do not override the input data if there is only one input data,
- // since it will clear all caches. Rely on wipe_cache to
- // clear caches
- continue;
- }
- caffe2::TensorProto* tensor_proto =
- tensor_kv.second.mutable_protos(iteration % protos_size);
- BlobSetTensor(blob, deserializer.Deserialize(*tensor_proto));
- // todo: for other types
- }
-}
-
-void runNetwork(
- shared_ptr<caffe2::Workspace> workspace,
- caffe2::NetBase* net,
- map<string, caffe2::TensorProtos>& tensor_protos_map,
- const bool wipe_cache,
- const bool run_individual,
- const bool run_on_gpu,
- const bool text_output,
- const int warmup,
- const int iter,
- const int num_blobs,
- const int sleep_before_run,
- const int sleep_between_iteration,
- const int sleep_between_net_and_operator,
- const std::string& output,
- const std::string& output_folder) {
-
- LOG(INFO) << "Starting benchmark.";
- caffe2::ObserverConfig::initSampleRate(1, 1, 1, run_individual, warmup);
- LOG(INFO) << "Running warmup runs.";
- for (int i = 0; i < warmup; ++i) {
- fillInputBlob(workspace, tensor_protos_map, i);
- CAFFE_ENFORCE(net->Run(), "Warmup run ", i, " has failed.");
- }
-
- if (wipe_cache) {
- caffe2::wipe_cache();
- }
- if (sleep_before_run > 0) {
- std::this_thread::sleep_for(std::chrono::seconds(sleep_before_run));
- }
- LOG(INFO) << "Main runs.";
- CAFFE_ENFORCE(
- iter >= 0,
- "Number of main runs should be non negative, provided ",
- iter,
- ".");
- LOG(INFO) << "net runs.";
- long long duration_sum = 0;
- for (int i = 0; i < iter; ++i) {
- caffe2::ObserverConfig::initSampleRate(1, 1, 1, 0, warmup);
- fillInputBlob(workspace, tensor_protos_map, i);
- if (wipe_cache) {
- caffe2::wipe_cache();
- }
- auto start = std::chrono::high_resolution_clock::now();
- CAFFE_ENFORCE(net->Run(), "Main run ", i, " has failed.");
- auto stop = std::chrono::high_resolution_clock::now();
- auto duration = std::chrono::duration_cast<std::chrono::microseconds>(stop - start);
- duration_sum += duration.count();
- // Write the output for the first num_blobs times
- writeOutput(
- workspace,
- run_on_gpu,
- output,
- output_folder,
- text_output,
- i,
- num_blobs);
- if (wipe_cache) {
- caffe2::wipe_cache();
- }
- if (sleep_between_iteration > 0) {
- std::this_thread::sleep_for(
- std::chrono::seconds(sleep_between_iteration));
- }
- }
- std::cout << "Average Duration: " << (duration_sum/iter) << " us" << std::endl;
- if (run_individual) {
- LOG(INFO) << "operator runs.";
- if (sleep_between_net_and_operator > 0) {
- std::this_thread::sleep_for(
- std::chrono::seconds(sleep_between_net_and_operator));
- }
- for (int i = 0; i < iter; ++i) {
- caffe2::ObserverConfig::initSampleRate(1, 1, 1, 1, warmup);
- fillInputBlob(workspace, tensor_protos_map, i);
- CAFFE_ENFORCE(net->Run(), "Main run ", i, " with operator has failed.");
- if (wipe_cache) {
- caffe2::wipe_cache();
- }
- if (sleep_between_iteration > 0) {
- std::this_thread::sleep_for(
- std::chrono::seconds(sleep_between_iteration));
- }
- }
- }
-}
-
-void writeOutput(
- shared_ptr<caffe2::Workspace> workspace,
- const bool run_on_gpu,
- const string& output,
- const string& output_folder,
- const bool text_output,
- const int index,
- const int num_blobs) {
- if (output.size() == 0) {
- return;
- }
- string output_prefix = output_folder.size() ? output_folder + "/" : "";
- vector<string> output_names = caffe2::split(',', output);
- if (output == "*") {
- output_names = workspace->Blobs();
- }
- for (const string& name : output_names) {
- CAFFE_ENFORCE(
- workspace->HasBlob(name),
- "You requested a non-existing blob: ",
- name);
- if (text_output) {
- if (run_on_gpu) {
-#ifdef __CUDA_ARCH__
- writeTextOutput<caffe2::CUDAContext, caffe2::TensorCUDA>(
- workspace->GetBlob(name)->GetMutable<caffe2::TensorCUDA>(),
- output_prefix,
- name,
- index,
- num_blobs);
-#else
- CAFFE_THROW("Not support GPU.");
-#endif
- } else {
- writeTextOutput<caffe2::CPUContext, caffe2::TensorCPU>(
- BlobGetMutableTensor(workspace->GetBlob(name), caffe2::CPU),
- output_prefix,
- name,
- index,
- num_blobs);
- }
- } else {
- // Do not support multiple entries per blob.
- CAFFE_ENFORCE(
- index == 0,
- "Binary file only support one output.");
- string serialized = SerializeBlob(*workspace->GetBlob(name), name);
- string output_filename = output_prefix + name;
- caffe2::WriteStringToFile(serialized, output_filename.c_str());
- }
- }
-}
-
-void logBenchmarkResult(
- const std::string& type,
- const std::string& metric,
- const std::string& unit,
- const int value) {
- LOG(INFO) << caffe2::NetObserverReporterPrint::IDENTIFIER << "{"
- << "\"type\": \"" << type << "\", "
- << "\"metric\": \"" << metric << "\", "
- << "\"unit\": \"" << unit << "\", "
- << "\"value\": " << c10::to_string(value) << "}\n";
-}
-
-long getVirtualMemoryIfOptionEnabled(bool FLAGS_measure_memory) {
- if (FLAGS_measure_memory) {
-#if defined(TARGET_OS_IPHONE) || \
-defined(TARGET_OS_MAC) || \
-defined(TARGET_IPHONE_SIMULATOR)
- malloc_statistics_t stats = {0};
- malloc_zone_statistics(nullptr, &stats);
- return stats.size_allocated;
-#elif defined(_WIN32)
- PROCESS_MEMORY_COUNTERS_EX pmc;
- GetProcessMemoryInfo(
- GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc));
- return pmc.PrivateUsage;
-#else
- struct mallinfo info = mallinfo();
- return info.uordblks;
-#endif
- }
-
- return 0;
-}
-
-int benchmark(
- int argc,
- char* argv[],
- const string& FLAGS_backend,
- const string& FLAGS_init_net,
- const string& FLAGS_input,
- const string& FLAGS_input_dims,
- const string& FLAGS_input_file,
- const string& FLAGS_input_type,
- int FLAGS_iter,
- bool FLAGS_measure_memory,
- const string& FLAGS_net,
- const string& FLAGS_output,
- const string& FLAGS_output_folder,
- bool FLAGS_run_individual,
- int FLAGS_sleep_before_run,
- int FLAGS_sleep_between_iteration,
- int FLAGS_sleep_between_net_and_operator,
- bool FLAGS_text_output,
- int FLAGS_warmup,
- bool FLAGS_wipe_cache) {
- // Check arguments to be correct
- {
- // Need to check whether file exists, as the file reader does not assert if
- // file does not exist
- std::ifstream net_file(FLAGS_net);
- CAFFE_ENFORCE(net_file.good());
- net_file.close();
-
- std::ifstream init_net_file(FLAGS_init_net);
- CAFFE_ENFORCE(init_net_file.good());
- init_net_file.close();
-
- if (FLAGS_input_file.size() > 0) {
- vector<string> input_files = caffe2::split(',', FLAGS_input_file);
- for (auto input_file : input_files) {
- std::ifstream ifile(input_file);
- CAFFE_ENFORCE(ifile.good());
- ifile.close();
- }
- }
- }
-
- observerConfig();
- caffe2::ShowLogInfoToStderr();
-
- auto workspace = std::make_shared<caffe2::Workspace>(new caffe2::Workspace());
- bool run_on_gpu = backendCudaSet(FLAGS_backend);
- // Run initialization network, measure resources used.
- long init_vmem = getVirtualMemoryIfOptionEnabled(FLAGS_measure_memory);
- caffe2::NetDef init_net_def;
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &init_net_def));
- setOperatorEngine(&init_net_def, FLAGS_backend);
- CAFFE_ENFORCE(workspace->RunNetOnce(init_net_def));
- init_vmem = getVirtualMemoryIfOptionEnabled(FLAGS_measure_memory) - init_vmem;
-
- map<string, caffe2::TensorProtos> tensor_protos_map;
- int num_blobs = loadInput(
- workspace,
- run_on_gpu,
- tensor_protos_map,
- FLAGS_input,
- FLAGS_input_file,
- FLAGS_input_dims,
- FLAGS_input_type);
-
- // Run main network.
- long predict_vmem = getVirtualMemoryIfOptionEnabled(FLAGS_measure_memory);
- caffe2::NetDef net_def;
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_net, &net_def));
- setOperatorEngine(&net_def, FLAGS_backend);
- if (!net_def.has_name()) {
- net_def.set_name("benchmark");
- }
- caffe2::NetBase* net = workspace->CreateNet(net_def);
- TORCH_CHECK_NOTNULL(net);
- runNetwork(
- workspace,
- net,
- tensor_protos_map,
- FLAGS_wipe_cache,
- FLAGS_run_individual,
- run_on_gpu,
- FLAGS_text_output,
- FLAGS_warmup,
- FLAGS_iter,
- num_blobs,
- FLAGS_sleep_before_run,
- FLAGS_sleep_between_iteration,
- FLAGS_sleep_between_net_and_operator,
- FLAGS_output,
- FLAGS_output_folder);
- predict_vmem = getVirtualMemoryIfOptionEnabled(
- FLAGS_measure_memory) - predict_vmem;
- if (FLAGS_measure_memory) {
- logBenchmarkResult(
- "NET_", "memory", "kB", (init_vmem + predict_vmem) / 1024);
- }
-
- return 0;
-}
diff --git a/binaries/benchmark_helper.h b/binaries/benchmark_helper.h
deleted file mode 100644
index bd48be7..0000000
--- a/binaries/benchmark_helper.h
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#pragma once
-
-#include <string>
-
-#include "caffe2/core/blob_serialization.h"
-#include "caffe2/core/init.h"
-#include "caffe2/core/logging.h"
-#include "caffe2/core/net.h"
-#include "caffe2/core/operator.h"
-#include "caffe2/utils/string_utils.h"
-#include "c10/util/string_utils.h"
-#include <c10/util/irange.h>
-
-using std::map;
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-template <typename ContextType, typename TensorType>
-void writeTextOutput(
- TensorType* tensor,
- const string& output_prefix,
- const string& name,
- int index,
- int num_blobs) {
- if (index >= num_blobs) {
- return;
- }
- string filename = name;
- std::replace(filename.begin(), filename.end(), '/', '_');
- string output_name = output_prefix + "/" + filename + ".txt";
- caffe2::TensorSerializer ser;
- caffe2::BlobProto blob_proto;
-
- ser.Serialize(
- *tensor, output_name, blob_proto.mutable_tensor(), 0, tensor->numel());
- blob_proto.set_name(output_name);
- blob_proto.set_type("Tensor");
- CAFFE_ENFORCE(blob_proto.has_tensor());
- caffe2::TensorProto tensor_proto = blob_proto.tensor();
- int dims_size = tensor_proto.dims_size();
- long long elem_dim_size =
- dims_size > 1 ? tensor_proto.dims(1) : tensor_proto.dims(0);
- for (const auto i : c10::irange(2, dims_size)) {
- elem_dim_size *= tensor_proto.dims(i);
- }
- std::vector<std::string> lines;
- std::string dims;
- for (const auto i : c10::irange(dims_size)) {
- int dim = tensor_proto.dims(i);
- if (i > 0) {
- dims += ", ";
- }
- dims += c10::to_string(dim);
- }
- lines.push_back(dims);
- std::stringstream line;
- if (tensor_proto.data_type() == caffe2::TensorProto::FLOAT) {
- auto start = tensor_proto.float_data().begin();
- auto end = tensor_proto.float_data().end();
- copy(start, end, std::ostream_iterator<float>(line, ","));
- } else if (tensor_proto.data_type() == caffe2::TensorProto::INT32) {
- auto start = tensor_proto.int32_data().begin();
- auto end = tensor_proto.int32_data().end();
- copy(start, end, std::ostream_iterator<int>(line, ","));
- } else {
- CAFFE_THROW("Unimplemented Blob type.");
- }
- // remove the last ,
- string str = line.str();
- if(str.length() != 0) {
- str.pop_back();
- }
- lines.push_back(str);
-
- // static casts are workaround for MSVC build
- auto flags = static_cast<std::ios_base::openmode>(std::ios::out);
- if (index != 0) {
- flags |= static_cast<std::ios_base::openmode>(std::ios::app);
- } else {
- flags |= static_cast<std::ios_base::openmode>(std::ios::trunc);
- }
- std::ofstream output_file(output_name, flags);
- std::ostream_iterator<std::string> output_iterator(output_file, "\n");
- std::copy(lines.begin(), lines.end(), output_iterator);
-}
-
-void observerConfig();
-bool backendCudaSet(const string&);
-void setDeviceType(caffe2::NetDef*, caffe2::DeviceType&);
-void setOperatorEngine(caffe2::NetDef*, const string&);
-int loadInput(
- shared_ptr<caffe2::Workspace> workspace,
- const bool run_on_gpu,
- map<string, caffe2::TensorProtos>& tensor_protos_map,
- const string& input,
- const string& input_file,
- const string& input_dims,
- const string& input_type);
-void fillInputBlob(
- shared_ptr<caffe2::Workspace> workspace,
- map<string, caffe2::TensorProtos>& tensor_protos_map,
- int iteration);
-void writeOutput(
- shared_ptr<caffe2::Workspace> workspace,
- const bool run_on_gpu,
- const string& output,
- const string& output_folder,
- const bool text_output,
- const int index,
- const int num_blobs);
-void logBenchmarkResult(
- const std::string& type,
- const std::string& metric,
- const std::string& unit,
- const int value);
-long getVirtualMemoryIfOptionEnabled(bool FLAGS_measure_memory);
-void runNetwork(
- shared_ptr<caffe2::Workspace> workspace,
- caffe2::NetBase* net,
- map<string, caffe2::TensorProtos>& tensor_protos_map,
- const bool wipe_cache,
- const bool run_individual,
- const bool run_on_gpu,
- const bool text_output,
- const int warmup,
- const int iter,
- const int num_blobs,
- const int sleep_before_run,
- const int sleep_between_iteration,
- const int sleep_between_net_and_operator,
- const std::string& output,
- const std::string& output_folder);
-int benchmark(
- int argc,
- char* argv[],
- const string& FLAGS_backend,
- const string& FLAGS_init_net,
- const string& FLAGS_input,
- const string& FLAGS_input_dims,
- const string& FLAGS_input_file,
- const string& FLAGS_input_type,
- int FLAGS_iter,
- bool FLAGS_measure_memory,
- const string& FLAGS_net,
- const string& FLAGS_output,
- const string& FLAGS_output_folder,
- bool FLAGS_run_individual,
- int FLAGS_sleep_before_run,
- int FLAGS_sleep_between_iteration,
- int FLAGS_sleep_between_net_and_operator,
- bool FLAGS_text_output,
- int FLAGS_warmup,
- bool FLAGS_wipe_cache);
diff --git a/binaries/caffe2_benchmark.cc b/binaries/caffe2_benchmark.cc
deleted file mode 100644
index 65eb7b7..0000000
--- a/binaries/caffe2_benchmark.cc
+++ /dev/null
@@ -1,32 +0,0 @@
-#include <fstream>
-#include <iterator>
-#include <string>
-
-#include "binaries/benchmark_args.h"
-#include "binaries/benchmark_helper.h"
-
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- benchmark(
- argc,
- argv,
- FLAGS_backend,
- FLAGS_init_net,
- FLAGS_input,
- FLAGS_input_dims,
- FLAGS_input_file,
- FLAGS_input_type,
- FLAGS_iter,
- FLAGS_measure_memory,
- FLAGS_net,
- FLAGS_output,
- FLAGS_output_folder,
- FLAGS_run_individual,
- FLAGS_sleep_before_run,
- FLAGS_sleep_between_iteration,
- FLAGS_sleep_between_net_and_operator,
- FLAGS_text_output,
- FLAGS_warmup,
- FLAGS_wipe_cache);
-}
diff --git a/binaries/convert_and_benchmark.cc b/binaries/convert_and_benchmark.cc
deleted file mode 100644
index ceb029c..0000000
--- a/binaries/convert_and_benchmark.cc
+++ /dev/null
@@ -1,770 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <opencv2/opencv.hpp>
-#include <cmath>
-#include <fstream>
-
-#include "caffe2/core/common.h"
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/core/logging.h"
-#include "caffe2/core/timer.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/utils/proto_utils.h"
-#include "caffe2/utils/string_utils.h"
-#include "caffe2/utils/bench_utils.h"
-
-#include "binaries/benchmark_args.h"
-#include "binaries/benchmark_helper.h"
-
-#include <observers/net_observer_reporter_print.h>
-#include <observers/observer_config.h>
-#include <observers/perf_observer.h>
-
-
-C10_DEFINE_int(
- batch_size,
- -1,
- "Specify the batch size of the input. The number of items in the "
- "input needs to be multiples of the batch size. If the batch size "
- "is less than 0, all inputs are in one batch.")
-C10_DEFINE_bool(color, true, "If set, load images in color.");
-C10_DEFINE_string(
- crop,
- "-1,-1",
- "The center cropped hight and width. If the value is less than zero, "
- "it is not cropped.");
-C10_DEFINE_string(input_image_files, "", "Files containing imput images");
-C10_DEFINE_string(input_text_files, "", "Text files to be written to blobs");
-C10_DEFINE_string(
- preprocess,
- "",
- "Options to specify the preprocess routines. The available options are "
- "subtract128, normalize, mean, std, bgrtorgb. If multiple steps are provided, they "
- "are separated by comma (,) in sequence.");
-C10_DEFINE_string(
- report_time,
- "",
- "Report the conversion stage time to screen. "
- "The format of the string is <type>|<identifier>. "
- "The valid type is 'json'. "
- "The valid identifier is nothing or an identifier that prefix every line");
-C10_DEFINE_string(
- scale,
- "-1,-1",
- "Scale the images to be within the min,max box. The shorter edge is "
- "min pixels. But if the other edge is more than the max pixels, the "
- "other edge and scaled to max pixels (and the shorter edge can be less "
- "than the min pixels");
-C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
-
-namespace caffe2 {
-
-void reportTime(
- std::string type,
- double ts,
- std::string metric,
- std::string unit) {
- if (FLAGS_report_time == "") {
- return;
- }
- vector<string> s = caffe2::split('|', FLAGS_report_time);
- assert(s[0] == "json");
- std::string identifier = "";
- if (s.size() > 1) {
- identifier = s[1];
- }
- std::cout << identifier << "{\"type\": \"" << type << "\", \"value\": " << ts
- << ", \"metric\": \"" << metric << "\", \"unit\": \"" << unit
- << "\"}" << std::endl;
-}
-
-void splitSizes(const std::string& arg, int* ptr0, int* ptr1) {
- vector<string> sizes = caffe2::split(',', arg);
- if (sizes.size() == 2) {
- *ptr0 = std::stoi(sizes[0]);
- *ptr1 = std::stoi(sizes[1]);
- } else if (sizes.size() == 1) {
- *ptr0 = std::stoi(sizes[0]);
- *ptr1 = std::stoi(sizes[0]);
- } else {
- assert(false);
- }
-}
-
-
-cv::Mat resizeImage(cv::Mat& img) {
- int min_size, max_size;
- splitSizes(FLAGS_scale, &min_size, &max_size);
- if ((min_size <= 0) && (max_size <= 0)) {
- return img;
- }
- if (max_size < 0) {
- max_size = INT_MAX;
- }
- assert(min_size <= max_size);
-
- int im_min_size = img.rows > img.cols ? img.cols : img.rows;
- int im_max_size = img.rows > img.cols ? img.rows : img.cols;
-
- double im_scale = 1.0 * min_size / im_min_size;
- if (im_scale * im_max_size > max_size) {
- im_scale = 1.0 * max_size / im_max_size;
- }
- int scaled_width = int(round(img.cols * im_scale));
- int scaled_height = int(round(img.rows * im_scale));
- assert((scaled_width <= max_size) && (scaled_height <= max_size));
- if ((scaled_width < min_size) || (scaled_height < min_size)) {
- assert((scaled_width == max_size) || (scaled_height == max_size));
- } else {
- assert((scaled_width == min_size) || (scaled_height == min_size));
- }
- cv::Mat resized_img;
- cv::resize(
- img,
- resized_img,
- cv::Size(),
- im_scale,
- im_scale,
- cv::INTER_LINEAR);
- return resized_img;
-}
-
-cv::Mat cropToRec(cv::Mat& img, int* height_ptr, int* width_ptr) {
- int height = *height_ptr;
- int width = *width_ptr;
- if ((height > 0) && (width > 0) &&
- ((img.rows != height) || (img.cols != width))) {
- cv::Mat cropped_img, cimg;
- cv::Rect roi;
- roi.x = int((img.cols - width) / 2);
- roi.y = int((img.rows - height) / 2);
- roi.x = roi.x < 0 ? 0 : roi.x;
- roi.y = roi.y < 0 ? 0 : roi.y;
- width = width > img.cols ? img.cols : width;
- height = height > img.rows ? img.rows : height;
- roi.width = width;
- roi.height = height;
- assert(
- 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= img.cols &&
- 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= img.rows);
- cropped_img = img(roi);
- // Make the image in continuous space in memory
- cimg = cropped_img.clone();
- *height_ptr = height;
- *width_ptr = width;
- return cimg;
- } else {
- return img;
- }
-}
-
-std::vector<float> convertToVector(cv::Mat& img) {
- std::vector<float> normalize(3, 1);
- std::vector<float> mean(3, 0);
- std::vector<float> std(3, 1);
- bool bgrtorgb = false;
- int size = img.cols * img.rows;
- vector<string> steps = caffe2::split(',', FLAGS_preprocess);
- for (int i = 0; i < steps.size(); i++) {
- auto step = steps[i];
- if (step == "subtract128") {
- mean = {128, 128, 128};
- std = {1, 1, 1};
- normalize = {1, 1, 1};
- } else if (step == "normalize") {
- normalize = {255, 255, 255};
- } else if (step == "mean") {
- mean = {0.406f, 0.456f, 0.485f};
- } else if (step == "std") {
- std = {0.225f, 0.224f, 0.229f};
- } else if (step == "bgrtorgb") {
- bgrtorgb = true;
- } else {
- CAFFE_ENFORCE(
- false,
- "Unsupported preprocess step. The supported steps are: subtract128, "
- "normalize,mean, std, swaprb.");
- }
- }
-
- int C = FLAGS_color ? 3 : 1;
- int total_size = C * size;
- std::vector<float> values(total_size);
- if (C == 1) {
- cv::MatIterator_<float> it, end;
- int idx = 0;
- for (it = img.begin<float>(), end = img.end<float>(); it != end; ++it) {
- values[idx++] = (*it / normalize[0] - mean[0]) / std[0];
- }
- } else {
- int i = 0;
- cv::MatIterator_<cv::Vec3f> it, end;
- int b = bgrtorgb ? 2 : 0;
- int g = 1;
- int r = bgrtorgb ? 0 : 2;
- for (it = img.begin<cv::Vec3f>(), end = img.end<cv::Vec3f>(); it != end;
- ++it, i++) {
- values[i] = (((*it)[b] / normalize[0] - mean[0]) / std[0]);
- int offset = size + i;
- values[offset] = (((*it)[g] / normalize[1] - mean[1]) / std[1]);
- offset = size + offset;
- values[offset] = (((*it)[r] / normalize[2] - mean[2]) / std[2]);
- }
- }
- return values;
-}
-
-std::vector<float> convertOneImage(
- std::string& filename,
- int* height_ptr,
- int* width_ptr) {
- assert(filename[0] != '~');
-
- std::cout << "Converting " << filename << std::endl;
-
- // Load image
- cv::Mat img_uint8 = cv::imread(
-#if CV_MAJOR_VERSION <= 3
- filename, FLAGS_color ? CV_LOAD_IMAGE_COLOR : CV_LOAD_IMAGE_GRAYSCALE);
-#else
- filename, FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
-#endif
- caffe2::Timer timer;
- timer.Start();
- cv::Mat img;
- // Convert image to floating point values
- img_uint8.convertTo(img, CV_32F);
- // Resize image
- cv::Mat resized_img = resizeImage(img);
-
- int height, width;
- splitSizes(FLAGS_crop, &height, &width);
- if ((height <= 0) || (width <= 0)) {
- height = resized_img.rows;
- width = resized_img.cols;
- }
- cv::Mat crop = cropToRec(resized_img, &height, &width);
-
- // Assert we don't have to deal with alignment
- DCHECK(crop.isContinuous());
- assert(crop.rows == height);
- assert(crop.cols == width);
- std::vector<float> one_image_values = convertToVector(crop);
- *height_ptr = height;
- *width_ptr = width;
- double ts = timer.MicroSeconds();
- reportTime("image_preprocess", ts, "convert", "us");
- return one_image_values;
-}
-
-int getBatchSize(int num_items) {
- int batch_size = FLAGS_batch_size;
- if (batch_size < 0) {
- batch_size = num_items;
- } else {
- assert(num_items % batch_size == 0);
- }
- return batch_size;
-}
-
-TensorProtos writeValues(
- std::vector<std::vector<std::vector<float>>>& values,
- std::vector<std::vector<int>>& dims) {
-
- caffe2::Timer timer;
- timer.Start();
-
- assert(dims.size() == values.size());
- int num_batches = dims.size();
-
- TensorProtos protos;
- for (int k = 0; k < num_batches; k++) {
- TensorProto* data;
- data = protos.add_protos();
- data->set_data_type(TensorProto::FLOAT);
- auto one_dim = dims[k];
- for (int dim : one_dim) {
- data->add_dims(dim);
- }
- int batch_size = one_dim[0];
- long long int entry_size = 1;
- for (int i = 1; i < one_dim.size(); i++) {
- entry_size *= one_dim[i];
- }
-
- // Not optimized
- for (int i = 0; i < batch_size; i++) {
- assert(values[k][i].size() == entry_size);
- for (int j = 0; j < values[k][i].size(); j++) {
- data->add_float_data(values[k][i][j]);
- }
- }
- }
- double ts = timer.MicroSeconds();
- reportTime("preprocess", ts, "data_pack", "us");
-
- return protos;
-}
-
-TensorProtos convertImages(std::string& image_file) {
- vector<string> file_names;
- if (image_file != "") {
- std::ifstream infile(image_file);
- std::string line;
- while (std::getline(infile, line)) {
- vector<string> file_name = caffe2::split(',', line);
- string name;
- if (file_name.size() == 3) {
- name = file_name[2];
- } else {
- name = line;
- }
- file_names.push_back(name);
- }
- } else {
- TensorProtos proto;
- return proto;
- }
- int batch_size = getBatchSize(file_names.size());
- int num_batches = file_names.size() / batch_size;
- assert(file_names.size() == batch_size * num_batches);
- std::vector<std::vector<std::vector<float>>> values;
- std::vector<std::vector<int>> dims;
- int C = FLAGS_color ? 3 : 1;
- for (int k = 0; k < num_batches; k++) {
- std::vector<std::vector<float>> one_value;
- int height = -1;
- int width = -1;
- for (int i = 0; i < batch_size; i++) {
- int idx = k * batch_size + i;
- int one_height, one_width;
- std::vector<float> one_image_values =
- convertOneImage(file_names[idx], &one_height, &one_width);
- if (height < 0 && width < 0) {
- height = one_height;
- width = one_width;
- } else {
- assert(height == one_height);
- assert(width == one_width);
- }
- one_value.push_back(one_image_values);
- }
- vector<int> one_dim = {batch_size, C, height, width};
- dims.push_back(one_dim);
- values.push_back(one_value);
- }
- return writeValues(values, dims);
-}
-
-template <class TYPE>
-vector<TYPE> splitString(std::string& line) {
- vector<string> vector_str = caffe2::split(',', line);
- vector<TYPE> vector_int;
- for (string str : vector_str) {
- vector_int.push_back((TYPE)std::stod(str));
- }
- return vector_int;
-}
-
-/* Convert the values in a json file to blobs
- The format of the json file should be:
- <number of items>, <dim2>.... (dimensions of items)
- <entry>, <entry>, <entry>... (all entries in one item)
- <entry>, <entry>, <entry>...
- ....
-*/
-TensorProtos convertValues(std::string& file_name) {
- if (file_name == "") {
- TensorProtos proto;
- return proto;
- }
- std::ifstream infile(file_name);
- std::string line;
- std::getline(infile, line);
- vector<int> file_dims = splitString <int>(line);
- assert(file_dims.size() >= 2);
-
- int num_items = file_dims[0];
- int batch_size = getBatchSize(num_items);
- int num_batches = num_items / batch_size;
- assert(num_items == batch_size * num_batches);
- vector<string> lines;
- while (std::getline(infile, line)) {
- lines.push_back(line);
- }
- assert(lines.size() == num_items);
- std::vector<std::vector<std::vector<float>>> values;
- std::vector<std::vector<int>> dims;
- for (int i = 0; i < num_batches; i++) {
- std::vector<std::vector<float>> one_value;
- int num = -1;
- for (int j = 0; j < batch_size; j++) {
- int idx = i * batch_size + j;
- std::string line = lines[idx];
- vector<float> item = splitString<float>(line);
- if (num < 0) {
- num = item.size();
- } else {
- assert(num == item.size());
- }
- one_value.push_back(item);
- }
- vector<int> batch_dims = file_dims;
- batch_dims[0] = batch_size;
- dims.push_back(batch_dims);
- values.push_back(one_value);
- }
-
- return writeValues(values, dims);
-}
-
-} // namespace caffe2
-
-void observerConfig() {
- caffe2::ClearGlobalNetObservers();
- caffe2::AddGlobalNetObserverCreator([](caffe2::NetBase* subject) {
- return std::make_unique<caffe2::PerfNetObserver>(subject);
- });
- caffe2::ObserverConfig::setReporter(
- std::make_unique<caffe2::NetObserverReporterPrint>());
-}
-
-bool backendCudaSet(const string& backend) {
- bool run_on_gpu = false;
- if (backend == "cuda") {
-#ifdef __CUDA_ARCH__
- if (caffe2::HasCudaGPU()) {
- run_on_gpu = true;
- } else {
- CAFFE_THROW("NO GPU support on this host machine");
- }
-#else
- CAFFE_THROW("NO GPU support");
-#endif
- }
- return run_on_gpu;
-}
-
-void setOperatorEngine(caffe2::NetDef* net_def, const string& backend) {
- if (backend != "builtin") {
- string engine;
- if( backend == "nnpack" ) {
- engine = "NNPACK";
- } else if ( backend == "eigen" ) {
- engine = "EIGEN";
- } else if ( backend == "mkl" ) {
- engine = "MKLDNN";
- } else if ( backend == "cuda" ) {
- engine = "CUDA";
- } else if ( backend == "dnnlowp" ) {
- engine = "DNNLOWP";
- } else if ( backend == "dnnlowp_acc16" ) {
- engine = "DNNLOWP_ACC16";
- } else if ( backend == "default" ) {
- engine = "";
- } else {
- engine = "NONE";
- }
- CAFFE_ENFORCE(engine != "NONE", "Backend is not supported");
- for (int i = 0; i < net_def->op_size(); i++) {
- caffe2::OperatorDef* op_def = net_def->mutable_op(i);
- op_def->set_engine(engine);
- }
- }
-}
-
-void fillInputBlob(
- shared_ptr<caffe2::Workspace> workspace,
- map<string, caffe2::TensorProtos>& tensor_protos_map,
- int iteration) {
- if (tensor_protos_map.empty()) {
- return;
- }
- static caffe2::TensorDeserializer deserializer;
- for (auto& tensor_kv : tensor_protos_map) {
- caffe2::Blob* blob = workspace->GetBlob(tensor_kv.first);
- if (blob == nullptr) {
- blob = workspace->CreateBlob(tensor_kv.first);
- }
- // todo: support gpu and make this function a template
- int protos_size = tensor_kv.second.protos_size();
- if (protos_size == 1 && iteration > 0) {
- // Do not override the input data if there is only one input data,
- // since it will clear all caches. Rely on wipe_cache to
- // clear caches
- continue;
- }
- caffe2::TensorProto* tensor_proto =
- tensor_kv.second.mutable_protos(iteration % protos_size);
- BlobSetTensor(blob, deserializer.Deserialize(*tensor_proto));
- // todo: for other types
- }
-}
-
-void writeOutput(
- shared_ptr<caffe2::Workspace> workspace,
- const bool run_on_gpu,
- const string& output,
- const string& output_folder,
- const bool text_output,
- const int index,
- const int num_blobs) {
- if (output.size() == 0) {
- return;
- }
- string output_prefix = output_folder.size() ? output_folder + "/" : "";
- vector<string> output_names = caffe2::split(',', output);
- if (output == "*") {
- output_names = workspace->Blobs();
- }
- for (const string& name : output_names) {
- CAFFE_ENFORCE(
- workspace->HasBlob(name),
- "You requested a non-existing blob: ",
- name);
- if (text_output) {
- if (run_on_gpu) {
-#ifdef __CUDA_ARCH__
- writeTextOutput<caffe2::CUDAContext, caffe2::TensorCUDA>(
- workspace->GetBlob(name)->GetMutable<caffe2::TensorCUDA>(),
- output_prefix,
- name,
- index,
- num_blobs);
-#else
- CAFFE_THROW("Not support GPU.");
-#endif
- } else {
- writeTextOutput<caffe2::CPUContext, caffe2::TensorCPU>(
- BlobGetMutableTensor(workspace->GetBlob(name), caffe2::CPU),
- output_prefix,
- name,
- index,
- num_blobs);
- }
- } else {
- // Do not support multiple entries per blob.
- CAFFE_ENFORCE(
- index == 0,
- "Binary file only support one output.");
- string serialized = SerializeBlob(*workspace->GetBlob(name), name);
- string output_filename = output_prefix + name;
- caffe2::WriteStringToFile(serialized, output_filename.c_str());
- }
- }
-}
-
-void runNetwork(
- shared_ptr<caffe2::Workspace> workspace,
- caffe2::NetDef& net_def,
- map<string, caffe2::TensorProtos>& tensor_protos_map,
- const bool wipe_cache,
- const bool run_individual,
- const bool run_on_gpu,
- const bool text_output,
- const int warmup,
- const int iter,
- const int num_blobs,
- const int sleep_before_run,
- const int sleep_between_iteration,
- const int sleep_between_net_and_operator,
- const std::string& output,
- const std::string& output_folder) {
-
- if (!net_def.has_name()) {
- net_def.set_name("benchmark");
- }
-
- caffe2::NetBase* net = workspace->CreateNet(net_def);
- TORCH_CHECK_NOTNULL(net);
-
- LOG(INFO) << "Starting benchmark.";
- caffe2::ObserverConfig::initSampleRate(1, 1, 1, run_individual, warmup);
- LOG(INFO) << "Running warmup runs.";
- for (int i = 0; i < warmup; ++i) {
- fillInputBlob(workspace, tensor_protos_map, i);
- CAFFE_ENFORCE(net->Run(), "Warmup run ", i, " has failed.");
- }
-
- if (wipe_cache) {
- caffe2::wipe_cache();
- }
- if (sleep_before_run > 0) {
- std::this_thread::sleep_for(std::chrono::seconds(sleep_before_run));
- }
- LOG(INFO) << "Main runs.";
- CAFFE_ENFORCE(
- iter >= 0,
- "Number of main runs should be non negative, provided ",
- iter,
- ".");
- LOG(INFO) << "net runs.";
- for (int i = 0; i < iter; ++i) {
- caffe2::ObserverConfig::initSampleRate(1, 1, 1, 0, warmup);
- fillInputBlob(workspace, tensor_protos_map, i);
- if (wipe_cache) {
- caffe2::wipe_cache();
- }
- CAFFE_ENFORCE(net->Run(), "Main run ", i, " has failed.");
- // Write the output for the first num_blobs times
- writeOutput(
- workspace,
- run_on_gpu,
- output,
- output_folder,
- text_output,
- i,
- num_blobs);
- if (wipe_cache) {
- caffe2::wipe_cache();
- }
- if (sleep_between_iteration > 0) {
- std::this_thread::sleep_for(
- std::chrono::seconds(sleep_between_iteration));
- }
- }
- if (run_individual) {
- LOG(INFO) << "operator runs.";
- if (sleep_between_net_and_operator > 0) {
- std::this_thread::sleep_for(
- std::chrono::seconds(sleep_between_net_and_operator));
- }
- for (int i = 0; i < iter; ++i) {
- caffe2::ObserverConfig::initSampleRate(1, 1, 1, 1, warmup);
- fillInputBlob(workspace, tensor_protos_map, i);
- CAFFE_ENFORCE(net->Run(), "Main run ", i, " with operator has failed.");
- if (wipe_cache) {
- caffe2::wipe_cache();
- }
- if (sleep_between_iteration > 0) {
- std::this_thread::sleep_for(
- std::chrono::seconds(sleep_between_iteration));
- }
- }
- }
-}
-
-int benchmark(
- int argc,
- char* argv[],
- const string& FLAGS_backend,
- const string& FLAGS_init_net,
- const string& FLAGS_input_dims,
- int FLAGS_iter,
- const string& FLAGS_net,
- const string& FLAGS_output,
- const string& FLAGS_output_folder,
- bool FLAGS_run_individual,
- int FLAGS_sleep_before_run,
- int FLAGS_sleep_between_iteration,
- int FLAGS_sleep_between_net_and_operator,
- bool FLAGS_text_output,
- int FLAGS_warmup,
- bool FLAGS_wipe_cache) {
- // Check arguments to be correct
- {
- // Need to check whether file exists, as the file reader does not assert if
- // file does not exist
- std::ifstream net_file(FLAGS_net);
- CAFFE_ENFORCE(net_file.good());
- net_file.close();
-
- std::ifstream init_net_file(FLAGS_init_net);
- CAFFE_ENFORCE(init_net_file.good());
- init_net_file.close();
- }
-
- observerConfig();
- caffe2::ShowLogInfoToStderr();
-
- auto workspace = std::make_shared<caffe2::Workspace>(new caffe2::Workspace());
- bool run_on_gpu = backendCudaSet(FLAGS_backend);
- // Run initialization network.
- caffe2::NetDef init_net_def;
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &init_net_def));
- setOperatorEngine(&init_net_def, FLAGS_backend);
- CAFFE_ENFORCE(workspace->RunNetOnce(init_net_def));
-
- // Run main network.
- caffe2::NetDef net_def;
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_net, &net_def));
- setOperatorEngine(&net_def, FLAGS_backend);
-
- map<string, caffe2::TensorProtos> tensor_protos_map;
-
- int num_blobs;
- vector<string> images = caffe2::split(';', FLAGS_input_image_files);
- for (int i = 0; i < images.size(); ++i) {
- vector<string> mapping = caffe2::split(',', images[i]);
- caffe2::TensorProtos proto_images = caffe2::convertImages(mapping[1]);
- workspace->CreateBlob(mapping[0]);
- tensor_protos_map.insert(std::make_pair(mapping[0], proto_images));
- num_blobs = proto_images.protos_size();
- }
-
- vector<string> values = caffe2::split(';', FLAGS_input_text_files);
- for (int i = 0; i < values.size(); ++i) {
- vector<string> mapping = caffe2::split(',', values[i]);
- caffe2::TensorProtos proto_values = caffe2::convertValues(mapping[1]);
- workspace->CreateBlob(mapping[0]);
- tensor_protos_map.insert(std::make_pair(mapping[0], proto_values));
- num_blobs = proto_values.protos_size();
- }
-
- runNetwork(
- workspace,
- net_def,
- tensor_protos_map,
- FLAGS_wipe_cache,
- FLAGS_run_individual,
- run_on_gpu,
- FLAGS_text_output,
- FLAGS_warmup,
- FLAGS_iter,
- num_blobs,
- FLAGS_sleep_before_run,
- FLAGS_sleep_between_iteration,
- FLAGS_sleep_between_net_and_operator,
- FLAGS_output,
- FLAGS_output_folder);
-
- return 0;
-}
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- benchmark(
- argc,
- argv,
- FLAGS_backend,
- FLAGS_init_net,
- FLAGS_input_dims,
- FLAGS_iter,
- FLAGS_net,
- FLAGS_output,
- FLAGS_output_folder,
- FLAGS_run_individual,
- FLAGS_sleep_before_run,
- FLAGS_sleep_between_iteration,
- FLAGS_sleep_between_net_and_operator,
- FLAGS_text_output,
- FLAGS_warmup,
- FLAGS_wipe_cache);
-
- return 0;
-}
diff --git a/binaries/convert_caffe_image_db.cc b/binaries/convert_caffe_image_db.cc
deleted file mode 100644
index dca13d6..0000000
--- a/binaries/convert_caffe_image_db.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/proto/caffe2_legacy.pb.h"
-#include "caffe2/core/logging.h"
-
-C10_DEFINE_string(input_db, "", "The input db.");
-C10_DEFINE_string(input_db_type, "", "The input db type.");
-C10_DEFINE_string(output_db, "", "The output db.");
-C10_DEFINE_string(output_db_type, "", "The output db type.");
-C10_DEFINE_int(batch_size, 1000, "The write batch size.");
-
-using caffe2::db::Cursor;
-using caffe2::db::DB;
-using caffe2::db::Transaction;
-using caffe2::CaffeDatum;
-using caffe2::TensorProto;
-using caffe2::TensorProtos;
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
-
- std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
- FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
- std::unique_ptr<DB> out_db(caffe2::db::CreateDB(
- FLAGS_output_db_type, FLAGS_output_db, caffe2::db::NEW));
- std::unique_ptr<Cursor> cursor(in_db->NewCursor());
- std::unique_ptr<Transaction> transaction(out_db->NewTransaction());
- int count = 0;
- for (; cursor->Valid(); cursor->Next()) {
- CaffeDatum datum;
- CAFFE_ENFORCE(datum.ParseFromString(cursor->value()));
- TensorProtos protos;
- TensorProto* data = protos.add_protos();
- TensorProto* label = protos.add_protos();
- label->set_data_type(TensorProto::INT32);
- label->add_dims(1);
- label->add_int32_data(datum.label());
- if (datum.encoded()) {
- // This is an encoded image. we will copy over the data directly.
- data->set_data_type(TensorProto::STRING);
- data->add_dims(1);
- data->add_string_data(datum.data());
- } else {
- // float data not supported right now.
- CAFFE_ENFORCE_EQ(datum.float_data_size(), 0);
- std::vector<char> buffer_vec(datum.data().size());
- char* buffer = buffer_vec.data();
- // swap order from CHW to HWC
- int channels = datum.channels();
- int size = datum.height() * datum.width();
- CAFFE_ENFORCE_EQ(datum.data().size(), channels * size);
- for (int c = 0; c < channels; ++c) {
- char* dst = buffer + c;
- const char* src = datum.data().c_str() + c * size;
- for (int n = 0; n < size; ++n) {
- dst[n*channels] = src[n];
- }
- }
- data->set_data_type(TensorProto::BYTE);
- data->add_dims(datum.height());
- data->add_dims(datum.width());
- data->add_dims(datum.channels());
- data->set_byte_data(buffer, datum.data().size());
- }
- transaction->Put(cursor->key(), SerializeAsString_EnforceCheck(protos));
- if (++count % FLAGS_batch_size == 0) {
- transaction->Commit();
- LOG(INFO) << "Converted " << count << " items so far.";
- }
- }
- LOG(INFO) << "A total of " << count << " items processed.";
- return 0;
-}
diff --git a/binaries/convert_db.cc b/binaries/convert_db.cc
deleted file mode 100644
index 0fc73c3..0000000
--- a/binaries/convert_db.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/core/logging.h"
-
-C10_DEFINE_string(input_db, "", "The input db.");
-C10_DEFINE_string(input_db_type, "", "The input db type.");
-C10_DEFINE_string(output_db, "", "The output db.");
-C10_DEFINE_string(output_db_type, "", "The output db type.");
-C10_DEFINE_int(batch_size, 1000, "The write batch size.");
-
-using caffe2::db::Cursor;
-using caffe2::db::DB;
-using caffe2::db::Transaction;
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
-
- std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
- FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
- std::unique_ptr<DB> out_db(caffe2::db::CreateDB(
- FLAGS_output_db_type, FLAGS_output_db, caffe2::db::NEW));
- std::unique_ptr<Cursor> cursor(in_db->NewCursor());
- std::unique_ptr<Transaction> transaction(out_db->NewTransaction());
- int count = 0;
- for (; cursor->Valid(); cursor->Next()) {
- transaction->Put(cursor->key(), cursor->value());
- if (++count % FLAGS_batch_size == 0) {
- transaction->Commit();
- LOG(INFO) << "Converted " << count << " items so far.";
- }
- }
- LOG(INFO) << "A total of " << count << " items processed.";
- return 0;
-}
diff --git a/binaries/convert_encoded_to_raw_leveldb.cc b/binaries/convert_encoded_to_raw_leveldb.cc
deleted file mode 100644
index c0727d7..0000000
--- a/binaries/convert_encoded_to_raw_leveldb.cc
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This script converts an image dataset to leveldb.
-//
-// FLAGS_input_folder is the root folder that holds all the images, and
-// FLAGS_list_file should be a list of files as well as their labels, in
-// the format as
-// subfolder1/file1.JPEG 7
-// ....
-
-#include <opencv2/opencv.hpp>
-
-#include <fstream> // NOLINT(readability/streams)
-#include <memory>
-#include <random>
-#include <string>
-
-#include "caffe2/core/init.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/core/logging.h"
-#include "leveldb/db.h"
-#include "leveldb/write_batch.h"
-
-C10_DEFINE_string(input_db_name, "", "The input image file name.");
-C10_DEFINE_string(output_db_name, "", "The output training leveldb name.");
-C10_DEFINE_bool(color, true, "If set, load images in color.");
-C10_DEFINE_int(
- scale,
- 256,
- "If FLAGS_raw is set, scale all the images' shorter edge to the given "
- "value.");
-C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
-
-namespace caffe2 {
-
-
-void ConvertToRawDataset(
- const string& input_db_name, const string& output_db_name) {
- // input leveldb
- std::unique_ptr<leveldb::DB> input_db;
- LOG(INFO) << "Opening input leveldb " << input_db_name;
- {
- leveldb::Options options;
- options.create_if_missing = false;
- leveldb::DB* db_temp;
- leveldb::Status status = leveldb::DB::Open(
- options, input_db_name, &db_temp);
- CAFFE_ENFORCE(status.ok(), "Failed to open leveldb ", input_db_name, ".");
- input_db.reset(db_temp);
- }
-
- // output leveldb
- std::unique_ptr<leveldb::DB> output_db;
- std::unique_ptr<leveldb::WriteBatch> batch;
- LOG(INFO) << "Opening leveldb " << output_db_name;
- {
- leveldb::Options options;
- options.error_if_exists = true;
- options.create_if_missing = true;
- options.write_buffer_size = 268435456;
- leveldb::DB* db_temp;
- leveldb::Status status = leveldb::DB::Open(
- options, output_db_name, &db_temp);
- CAFFE_ENFORCE(
- status.ok(),
- "Failed to open leveldb ",
- output_db_name,
- ". Is it already existing?");
- output_db.reset(db_temp);
- }
- batch.reset(new leveldb::WriteBatch());
-
- TensorProtos input_protos;
- TensorProtos output_protos;
- TensorProto* data = output_protos.add_protos();
- TensorProto* label = output_protos.add_protos();
- data->set_data_type(TensorProto::BYTE);
- data->add_dims(0);
- data->add_dims(0);
- if (FLAGS_color) {
- data->add_dims(3);
- }
- string value;
-
- unique_ptr<leveldb::Iterator> iter;
- iter.reset(input_db->NewIterator(leveldb::ReadOptions()));
- iter->SeekToFirst();
- int count = 0;
- for (; iter->Valid(); iter->Next()) {
- CAFFE_ENFORCE(input_protos.ParseFromString(iter->value().ToString()));
- label->CopyFrom(input_protos.protos(1));
- const string& encoded_image = input_protos.protos(0).string_data(0);
- int encoded_size = encoded_image.size();
- cv::Mat img = cv::imdecode(
- cv::Mat(
- 1, &encoded_size, CV_8UC1, const_cast<char*>(encoded_image.data())),
- FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
- cv::Mat resized_img;
- int scaled_width, scaled_height;
- if (FLAGS_warp) {
- scaled_width = FLAGS_scale;
- scaled_height = FLAGS_scale;
- } else if (img.rows > img.cols) {
- scaled_width = FLAGS_scale;
- scaled_height = static_cast<float>(img.rows) * FLAGS_scale / img.cols;
- } else {
- scaled_height = FLAGS_scale;
- scaled_width = static_cast<float>(img.cols) * FLAGS_scale / img.rows;
- }
- cv::resize(img, resized_img, cv::Size(scaled_width, scaled_height), 0, 0,
- cv::INTER_LINEAR);
- data->set_dims(0, scaled_height);
- data->set_dims(1, scaled_width);
- DCHECK(resized_img.isContinuous());
- data->set_byte_data(
- resized_img.ptr(),
- scaled_height * scaled_width * (FLAGS_color ? 3 : 1));
- output_protos.SerializeToString(&value);
- // Put in db
- batch->Put(iter->key(), value);
- if (++count % 1000 == 0) {
- output_db->Write(leveldb::WriteOptions(), batch.get());
- batch.reset(new leveldb::WriteBatch());
- LOG(INFO) << "Processed " << count << " files.";
- }
- }
- // write the last batch
- if (count % 1000 != 0) {
- output_db->Write(leveldb::WriteOptions(), batch.get());
- }
- LOG(INFO) << "Processed a total of " << count << " files.";
-}
-
-} // namespace caffe2
-
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- caffe2::ConvertToRawDataset(FLAGS_input_db_name, FLAGS_output_db_name);
- return 0;
-}
diff --git a/binaries/convert_image_to_tensor.cc b/binaries/convert_image_to_tensor.cc
deleted file mode 100644
index 2cbfb07..0000000
--- a/binaries/convert_image_to_tensor.cc
+++ /dev/null
@@ -1,450 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <opencv2/opencv.hpp>
-#include <cmath>
-#include <fstream>
-
-#include "caffe2/core/common.h"
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/core/logging.h"
-#include "caffe2/core/timer.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/utils/proto_utils.h"
-#include "caffe2/utils/string_utils.h"
-
-
-C10_DEFINE_int(
- batch_size,
- -1,
- "Specify the batch size of the input. The number of items in the "
- "input needs to be multiples of the batch size. If the batch size "
- "is less than 0, all inputs are in one batch.")
-C10_DEFINE_bool(color, true, "If set, load images in color.");
-C10_DEFINE_string(
- crop,
- "-1,-1",
- "The center cropped hight and width. If the value is less than zero, "
- "it is not cropped.");
-C10_DEFINE_string(input_images, "", "Comma separated images");
-C10_DEFINE_string(input_image_file, "", "The file containing imput images");
-C10_DEFINE_string(input_text_file, "", "the text file to be written to blobs");
-C10_DEFINE_string(
- output_tensor,
- "",
- "The output tensor file in NCHW for input images");
-C10_DEFINE_string(
- output_text_tensor,
- "",
- "The output tensor file for the text input specified in input_text_file");
-C10_DEFINE_string(
- preprocess,
- "",
- "Options to specify the preprocess routines. The available options are "
- "subtract128, normalize, mean, std, bgrtorgb. If multiple steps are provided, they "
- "are separated by comma (,) in sequence.");
-C10_DEFINE_string(
- report_time,
- "",
- "Report the conversion stage time to screen. "
- "The format of the string is <type>|<identifier>. "
- "The valid type is 'json'. "
- "The valid identifier is nothing or an identifier that prefix every line");
-C10_DEFINE_string(
- scale,
- "-1,-1",
- "Scale the images to be within the min,max box. The shorter edge is "
- "min pixels. But if the other edge is more than the max pixels, the "
- "other edge and scaled to max pixels (and the shorter edge can be less "
- "than the min pixels");
-C10_DEFINE_bool(text_output, false, "Write the output in text format.");
-C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
-
-namespace caffe2 {
-
-void reportTime(
- std::string type,
- double ts,
- std::string metric,
- std::string unit) {
- if (FLAGS_report_time == "") {
- return;
- }
- vector<string> s = caffe2::split('|', FLAGS_report_time);
- assert(s[0] == "json");
- std::string identifier = "";
- if (s.size() > 1) {
- identifier = s[1];
- }
- std::cout << identifier << "{\"type\": \"" << type << "\", \"value\": " << ts
- << ", \"metric\": \"" << metric << "\", \"unit\": \"" << unit
- << "\"}" << std::endl;
-}
-
-void splitSizes(const std::string& arg, int* ptr0, int* ptr1) {
- vector<string> sizes = caffe2::split(',', arg);
- if (sizes.size() == 2) {
- *ptr0 = std::stoi(sizes[0]);
- *ptr1 = std::stoi(sizes[1]);
- } else if (sizes.size() == 1) {
- *ptr0 = std::stoi(sizes[0]);
- *ptr1 = std::stoi(sizes[0]);
- } else {
- assert(false);
- }
-}
-
-
-cv::Mat resizeImage(cv::Mat& img) {
- int min_size, max_size;
- splitSizes(FLAGS_scale, &min_size, &max_size);
- if ((min_size <= 0) && (max_size <= 0)) {
- return img;
- }
- if (max_size < 0) {
- max_size = INT_MAX;
- }
- assert(min_size <= max_size);
-
- int im_min_size = img.rows > img.cols ? img.cols : img.rows;
- int im_max_size = img.rows > img.cols ? img.rows : img.cols;
-
- double im_scale = 1.0 * min_size / im_min_size;
- if (im_scale * im_max_size > max_size) {
- im_scale = 1.0 * max_size / im_max_size;
- }
- int scaled_width = int(round(img.cols * im_scale));
- int scaled_height = int(round(img.rows * im_scale));
- assert((scaled_width <= max_size) && (scaled_height <= max_size));
- if ((scaled_width < min_size) || (scaled_height < min_size)) {
- assert((scaled_width == max_size) || (scaled_height == max_size));
- } else {
- assert((scaled_width == min_size) || (scaled_height == min_size));
- }
- cv::Mat resized_img;
- cv::resize(
- img,
- resized_img,
- cv::Size(),
- im_scale,
- im_scale,
- cv::INTER_LINEAR);
- return resized_img;
-}
-
-cv::Mat cropToRec(cv::Mat& img, int* height_ptr, int* width_ptr) {
- int height = *height_ptr;
- int width = *width_ptr;
- if ((height > 0) && (width > 0) &&
- ((img.rows != height) || (img.cols != width))) {
- cv::Mat cropped_img, cimg;
- cv::Rect roi;
- roi.x = int((img.cols - width) / 2);
- roi.y = int((img.rows - height) / 2);
- roi.x = roi.x < 0 ? 0 : roi.x;
- roi.y = roi.y < 0 ? 0 : roi.y;
- width = width > img.cols ? img.cols : width;
- height = height > img.rows ? img.rows : height;
- roi.width = width;
- roi.height = height;
- assert(
- 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= img.cols &&
- 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= img.rows);
- cropped_img = img(roi);
- // Make the image in continuous space in memory
- cimg = cropped_img.clone();
- *height_ptr = height;
- *width_ptr = width;
- return cimg;
- } else {
- return img;
- }
-}
-
-std::vector<float> convertToVector(cv::Mat& img) {
- std::vector<float> normalize(3, 1);
- std::vector<float> mean(3, 0);
- std::vector<float> std(3, 1);
- bool bgrtorgb = false;
- int size = img.cols * img.rows;
- vector<string> steps = caffe2::split(',', FLAGS_preprocess);
- for (int i = 0; i < steps.size(); i++) {
- auto step = steps[i];
- if (step == "subtract128") {
- mean = {128, 128, 128};
- std = {1, 1, 1};
- normalize = {1, 1, 1};
- } else if (step == "normalize") {
- normalize = {255, 255, 255};
- } else if (step == "mean") {
- mean = {0.406f, 0.456f, 0.485f};
- } else if (step == "std") {
- std = {0.225f, 0.224f, 0.229f};
- } else if (step == "bgrtorgb") {
- bgrtorgb = true;
- } else {
- CAFFE_ENFORCE(
- false,
- "Unsupported preprocess step. The supported steps are: subtract128, "
- "normalize,mean, std, swaprb.");
- }
- }
-
- int C = FLAGS_color ? 3 : 1;
- int total_size = C * size;
- std::vector<float> values(total_size);
- if (C == 1) {
- cv::MatIterator_<float> it, end;
- int idx = 0;
- for (it = img.begin<float>(), end = img.end<float>(); it != end; ++it) {
- values[idx++] = (*it / normalize[0] - mean[0]) / std[0];
- }
- } else {
- int i = 0;
- cv::MatIterator_<cv::Vec3f> it, end;
- int b = bgrtorgb ? 2 : 0;
- int g = 1;
- int r = bgrtorgb ? 0 : 2;
- for (it = img.begin<cv::Vec3f>(), end = img.end<cv::Vec3f>(); it != end;
- ++it, i++) {
- values[i] = (((*it)[b] / normalize[0] - mean[0]) / std[0]);
- int offset = size + i;
- values[offset] = (((*it)[g] / normalize[1] - mean[1]) / std[1]);
- offset = size + offset;
- values[offset] = (((*it)[r] / normalize[2] - mean[2]) / std[2]);
- }
- }
- return values;
-}
-
-std::vector<float> convertOneImage(
- std::string& filename,
- int* height_ptr,
- int* width_ptr) {
- assert(filename[0] != '~');
-
- std::cout << "Converting " << filename << std::endl;
-
- // Load image
- cv::Mat img_uint8 = cv::imread(
-#if CV_MAJOR_VERSION <= 3
- filename, FLAGS_color ? CV_LOAD_IMAGE_COLOR : CV_LOAD_IMAGE_GRAYSCALE);
-#else
- filename, FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
-#endif
- caffe2::Timer timer;
- timer.Start();
- cv::Mat img;
- // Convert image to floating point values
- img_uint8.convertTo(img, CV_32F);
- // Resize image
- cv::Mat resized_img = resizeImage(img);
-
- int height, width;
- splitSizes(FLAGS_crop, &height, &width);
- if ((height <= 0) || (width <= 0)) {
- height = resized_img.rows;
- width = resized_img.cols;
- }
- cv::Mat crop = cropToRec(resized_img, &height, &width);
-
- // Assert we don't have to deal with alignment
- DCHECK(crop.isContinuous());
- assert(crop.rows == height);
- assert(crop.cols == width);
- std::vector<float> one_image_values = convertToVector(crop);
- *height_ptr = height;
- *width_ptr = width;
- double ts = timer.MicroSeconds();
- reportTime("image_preprocess", ts, "convert", "us");
- return one_image_values;
-}
-
-int getBatchSize(int num_items) {
- int batch_size = FLAGS_batch_size;
- if (batch_size < 0) {
- batch_size = num_items;
- } else {
- assert(num_items % batch_size == 0);
- }
- return batch_size;
-}
-
-void writeValues(
- std::vector<std::vector<std::vector<float>>>& values,
- std::vector<std::vector<int>>& dims,
- std::string output_file) {
-
- caffe2::Timer timer;
- timer.Start();
-
- assert(dims.size() == values.size());
- int num_batches = dims.size();
-
- TensorProtos protos;
- for (int k = 0; k < num_batches; k++) {
- TensorProto* data;
- data = protos.add_protos();
- data->set_data_type(TensorProto::FLOAT);
- auto one_dim = dims[k];
- for (int dim : one_dim) {
- data->add_dims(dim);
- }
- int batch_size = one_dim[0];
- long long int entry_size = 1;
- for (int i = 1; i < one_dim.size(); i++) {
- entry_size *= one_dim[i];
- }
-
- // Not optimized
- for (int i = 0; i < batch_size; i++) {
- assert(values[k][i].size() == entry_size);
- for (int j = 0; j < values[k][i].size(); j++) {
- data->add_float_data(values[k][i][j]);
- }
- }
- }
- double ts = timer.MicroSeconds();
- reportTime("preprocess", ts, "data_pack", "us");
-
- if (FLAGS_text_output) {
- caffe2::WriteProtoToTextFile(protos, output_file);
- } else {
- caffe2::WriteProtoToBinaryFile(protos, output_file);
- }
-}
-
-void convertImages() {
- vector<string> file_names;
- if (FLAGS_input_images != "") {
- file_names = caffe2::split(',', FLAGS_input_images);
- } else if (FLAGS_input_image_file != "") {
- std::ifstream infile(FLAGS_input_image_file);
- std::string line;
- while (std::getline(infile, line)) {
- vector<string> file_name = caffe2::split(',', line);
- string name;
- if (file_name.size() == 3) {
- name = file_name[2];
- } else {
- name = line;
- }
- file_names.push_back(name);
- }
- } else {
- return;
- }
- int batch_size = getBatchSize(file_names.size());
- int num_batches = file_names.size() / batch_size;
- assert(file_names.size() == batch_size * num_batches);
- std::vector<std::vector<std::vector<float>>> values;
- std::vector<std::vector<int>> dims;
- int C = FLAGS_color ? 3 : 1;
- for (int k = 0; k < num_batches; k++) {
- std::vector<std::vector<float>> one_value;
- int height = -1;
- int width = -1;
- for (int i = 0; i < batch_size; i++) {
- int idx = k * batch_size + i;
- int one_height, one_width;
- std::vector<float> one_image_values =
- convertOneImage(file_names[idx], &one_height, &one_width);
- if (height < 0 && width < 0) {
- height = one_height;
- width = one_width;
- } else {
- assert(height == one_height);
- assert(width == one_width);
- }
- one_value.push_back(one_image_values);
- }
- vector<int> one_dim = {batch_size, C, height, width};
- dims.push_back(one_dim);
- values.push_back(one_value);
- }
- writeValues(values, dims, FLAGS_output_tensor);
-}
-
-template <class TYPE>
-vector<TYPE> splitString(std::string& line) {
- vector<string> vector_str = caffe2::split(',', line);
- vector<TYPE> vector_int;
- for (string str : vector_str) {
- vector_int.push_back((TYPE)std::stod(str));
- }
- return vector_int;
-}
-
-/* Convert the values in a json file to blobs
- The format of the json file should be:
- <number of items>, <dim2>.... (dimensions of items)
- <entry>, <entry>, <entry>... (all entries in one item)
- <entry>, <entry>, <entry>...
- ....
-*/
-void convertValues() {
- if (FLAGS_input_text_file == "") {
- return;
- }
- std::ifstream infile(FLAGS_input_text_file);
- std::string line;
- std::getline(infile, line);
- vector<int> file_dims = splitString <int>(line);
- assert(file_dims.size() >= 2);
-
- int num_items = file_dims[0];
- int batch_size = getBatchSize(num_items);
- int num_batches = num_items / batch_size;
- assert(num_items == batch_size * num_batches);
- vector<string> lines;
- while (std::getline(infile, line)) {
- lines.push_back(line);
- }
- assert(lines.size() == num_items);
- std::vector<std::vector<std::vector<float>>> values;
- std::vector<std::vector<int>> dims;
- for (int i = 0; i < num_batches; i++) {
- std::vector<std::vector<float>> one_value;
- int num = -1;
- for (int j = 0; j < batch_size; j++) {
- int idx = i * batch_size + j;
- std::string line = lines[idx];
- vector<float> item = splitString<float>(line);
- if (num < 0) {
- num = item.size();
- } else {
- assert(num == item.size());
- }
- one_value.push_back(item);
- }
- vector<int> batch_dims = file_dims;
- batch_dims[0] = batch_size;
- dims.push_back(batch_dims);
- values.push_back(one_value);
- }
-
- writeValues(values, dims, FLAGS_output_text_tensor);
-}
-
-} // namespace caffe2
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- caffe2::convertImages();
- caffe2::convertValues();
- return 0;
-}
diff --git a/binaries/db_throughput.cc b/binaries/db_throughput.cc
deleted file mode 100644
index 851447a..0000000
--- a/binaries/db_throughput.cc
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <cstdio>
-#include <thread>
-#include <vector>
-
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/core/timer.h"
-#include "caffe2/core/logging.h"
-
-C10_DEFINE_string(input_db, "", "The input db.");
-C10_DEFINE_string(input_db_type, "", "The input db type.");
-C10_DEFINE_int(report_interval, 1000, "The report interval.");
-C10_DEFINE_int(repeat, 10, "The number to repeat the throughput test.");
-C10_DEFINE_bool(use_reader, false, "If true, use the reader interface.");
-C10_DEFINE_int(
- num_read_threads,
- 1,
- "The number of concurrent reading threads.");
-
-using caffe2::db::Cursor;
-using caffe2::db::DB;
-using caffe2::db::DBReader;
-using caffe2::string;
-
-void TestThroughputWithDB() {
- std::unique_ptr<DB> in_db(caffe2::db::CreateDB(
- FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ));
- std::unique_ptr<Cursor> cursor(in_db->NewCursor());
- for (int iter_id = 0; iter_id < FLAGS_repeat; ++iter_id) {
- caffe2::Timer timer;
- for (int i = 0; i < FLAGS_report_interval; ++i) {
- string key = cursor->key();
- string value = cursor->value();
- //VLOG(1) << "Key " << key;
- cursor->Next();
- if (!cursor->Valid()) {
- cursor->SeekToFirst();
- }
- }
- double elapsed_seconds = timer.Seconds();
- printf(
- "Iteration %03d, took %4.5f seconds, throughput %f items/sec.\n",
- iter_id,
- elapsed_seconds,
- FLAGS_report_interval / elapsed_seconds);
- }
-}
-
-void TestThroughputWithReaderWorker(const DBReader* reader, int thread_id) {
- string key, value;
- for (int iter_id = 0; iter_id < FLAGS_repeat; ++iter_id) {
- caffe2::Timer timer;
- for (int i = 0; i < FLAGS_report_interval; ++i) {
- reader->Read(&key, &value);
- }
- double elapsed_seconds = timer.Seconds();
- printf(
- "Thread %03d iteration %03d, took %4.5f seconds, "
- "throughput %f items/sec.\n",
- thread_id,
- iter_id,
- elapsed_seconds,
- FLAGS_report_interval / elapsed_seconds);
- }
-}
-
-void TestThroughputWithReader() {
- caffe2::db::DBReader reader(FLAGS_input_db_type, FLAGS_input_db);
- std::vector<std::unique_ptr<std::thread>> reading_threads(
- FLAGS_num_read_threads);
- for (int i = 0; i < reading_threads.size(); ++i) {
- reading_threads[i].reset(new std::thread(
- TestThroughputWithReaderWorker, &reader, i));
- }
- for (int i = 0; i < reading_threads.size(); ++i) {
- reading_threads[i]->join();
- }
-}
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- if (FLAGS_use_reader) {
- TestThroughputWithReader();
- } else {
- TestThroughputWithDB();
- }
- return 0;
-}
diff --git a/binaries/intra_inter_benchmark.cc b/binaries/intra_inter_benchmark.cc
deleted file mode 100644
index 0f191dc..0000000
--- a/binaries/intra_inter_benchmark.cc
+++ /dev/null
@@ -1,166 +0,0 @@
-#include "ATen/ATen.h"
-#include "ATen/Parallel.h"
-
-#include "c10/util/Flags.h"
-#include "caffe2/core/init.h"
-
-#include <chrono>
-#include <condition_variable>
-#include <ctime>
-#include <iostream>
-#include <mutex>
-#include <thread>
-
-C10_DEFINE_int(iter_pow, 10, "Number of tasks, 2^N");
-C10_DEFINE_int(sub_iter, 1024, "Number of subtasks");
-C10_DEFINE_int(warmup_iter_pow, 3, "Number of warmup tasks, 2^N");
-C10_DEFINE_int(inter_op_threads, 0, "Number of inter-op threads");
-C10_DEFINE_int(intra_op_threads, 0, "Number of intra-op threads");
-C10_DEFINE_int(tensor_dim, 50, "Tensor dim");
-C10_DEFINE_int(benchmark_iter, 10, "Number of times to run benchmark")
-C10_DEFINE_bool(extra_stats, false,
- "Collect extra stats; warning: skews results");
-C10_DEFINE_string(task_type, "add", "Tensor operation: add or mm");
-
-namespace {
-std::atomic<int> counter{0};
-int overall_tasks = 0;
-std::condition_variable cv;
-std::mutex tasks_mutex;
-bool run_mm = false;
-
-std::mutex stats_mutex;
-std::unordered_set<std::thread::id> tids;
-}
-
-void wait() {
- std::unique_lock<std::mutex> lk(tasks_mutex);
- while (counter < overall_tasks) {
- cv.wait(lk);
- }
-}
-
-void _launch_tasks_tree(
- int level, int end_level, at::Tensor& left, at::Tensor& right) {
- if (level == end_level) {
- at::parallel_for(0, FLAGS_sub_iter, 1,
- [&left, &right](int64_t begin, int64_t end) {
- if (FLAGS_extra_stats) {
- std::unique_lock<std::mutex> lk(stats_mutex);
- tids.insert(std::this_thread::get_id());
- }
- for (auto k = begin; k < end; ++k) {
- if (run_mm) {
- left.mm(right);
- } else {
- left.add(right);
- }
- auto cur_ctr = ++counter;
- if (cur_ctr == overall_tasks) {
- std::unique_lock<std::mutex> lk(tasks_mutex);
- cv.notify_one();
- }
- }
- });
- } else {
- at::launch([&left, &right, level, end_level]() {
- _launch_tasks_tree(level + 1, end_level, left, right);
- });
- at::launch([&left, &right, level, end_level]() {
- _launch_tasks_tree(level + 1, end_level, left, right);
- });
- }
-};
-
-void launch_tasks_and_wait(at::Tensor& left, at::Tensor& right, int iter_pow) {
- overall_tasks = pow(2, iter_pow) * FLAGS_sub_iter;
- counter = 0;
-
- _launch_tasks_tree(0, iter_pow, left, right);
- wait();
-}
-
-void reset_extra_stats() {
- tids.clear();
-}
-
-void print_extra_stats() {
- std::cout << "# threads: " << tids.size() << std::endl;
-}
-
-void print_runtime_stats(const std::vector<float>& runtimes) {
- TORCH_INTERNAL_ASSERT(!runtimes.empty());
- float sum = 0.0;
- float sqr_sum = 0.0;
- size_t N = runtimes.size();
- for (size_t idx = 0; idx < N; ++idx) {
- sum += runtimes[idx];
- sqr_sum += runtimes[idx] * runtimes[idx];
- }
- float mean = sum / N;
- float sd = std::sqrt(sqr_sum / N - mean * mean);
- std::cout << "N = " << N << ", mean = " << mean << ", sd = " << sd
- << std::endl;
-}
-
-int main(int argc, char** argv) {
- if (!c10::ParseCommandLineFlags(&argc, &argv)) {
- std::cout << "Failed to parse command line flags" << std::endl;
- return -1;
- }
- caffe2::unsafeRunCaffe2InitFunction("registerThreadPools");
- at::init_num_threads();
-
- if (FLAGS_inter_op_threads > 0) {
- at::set_num_interop_threads(FLAGS_inter_op_threads);
- }
- if (FLAGS_intra_op_threads > 0) {
- at::set_num_threads(FLAGS_intra_op_threads);
- }
-
- TORCH_CHECK(FLAGS_task_type == "add" || FLAGS_task_type == "mm");
- run_mm = FLAGS_task_type == "mm";
-
- auto left = at::ones({FLAGS_tensor_dim, FLAGS_tensor_dim}, at::kFloat);
- auto right = at::ones({FLAGS_tensor_dim, FLAGS_tensor_dim}, at::kFloat);
-
- std::cout << "Launching " << pow(2, FLAGS_warmup_iter_pow)
- << " warmup tasks" << std::endl;
-
- typedef std::chrono::high_resolution_clock clock;
- typedef std::chrono::milliseconds ms;
-
- std::chrono::time_point<clock> start_time = clock::now();
- launch_tasks_and_wait(left, right, FLAGS_warmup_iter_pow);
- auto duration = static_cast<float>(
- std::chrono::duration_cast<ms>(clock::now() - start_time).count());
-
- std::cout << "Warmup time: " << duration << " ms." << std::endl;
-
- std::cout << "Launching " << pow(2, FLAGS_iter_pow) << " tasks with "
- << FLAGS_sub_iter << " subtasks each, using "
- << at::get_num_interop_threads() << " inter-op threads and "
- << at::get_num_threads() << " intra-op threads, "
- << "tensor dim: " << FLAGS_tensor_dim
- << ", task type: " << FLAGS_task_type << std::endl;
-
- std::vector<float> runtimes;
- for (auto bench_iter = 0; bench_iter < FLAGS_benchmark_iter; ++bench_iter) {
- reset_extra_stats();
- start_time = clock::now();
- launch_tasks_and_wait(left, right, FLAGS_iter_pow);
- duration = static_cast<float>(
- std::chrono::duration_cast<ms>(clock::now() - start_time).count());
- runtimes.push_back(duration);
-
- if (FLAGS_extra_stats) {
- print_extra_stats();
- }
-
- std::cout << "Runtime: " << duration << " ms." << std::endl;
- }
-
- print_runtime_stats(runtimes);
-
- return 0;
-}
diff --git a/binaries/make_cifar_db.cc b/binaries/make_cifar_db.cc
deleted file mode 100644
index 3c62710..0000000
--- a/binaries/make_cifar_db.cc
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-//
-// This script converts the CIFAR dataset to the leveldb format used
-// by caffe to perform classification.
-// Usage:
-// convert_cifar_data input_folder output_db_file
-// The CIFAR dataset could be downloaded at
-// http://www.cs.toronto.edu/~kriz/cifar.html
-
-#include <array>
-#include <fstream> // NOLINT(readability/streams)
-#include <sstream>
-#include <string>
-
-#include "caffe2/core/common.h"
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/core/logging.h"
-
-C10_DEFINE_string(input_folder, "", "The input folder name.");
-C10_DEFINE_string(output_train_db_name, "", "The output training db name.");
-C10_DEFINE_string(output_test_db_name, "", "The output testing db name.");
-C10_DEFINE_string(db, "leveldb", "The db type.");
-C10_DEFINE_bool(
- is_cifar100,
- false,
- "If set, convert cifar100. Otherwise do cifar10.");
-
-namespace caffe2 {
-
-using std::stringstream;
-
-const int kCIFARSize = 32;
-const int kCIFARImageNBytes = kCIFARSize * kCIFARSize * 3;
-const int kCIFAR10BatchSize = 10000;
-const int kCIFAR10TestDataSize = 10000;
-const int kCIFAR10TrainBatches = 5;
-
-const int kCIFAR100TrainDataSize = 50000;
-const int kCIFAR100TestDataSize = 10000;
-
-void ReadImage(std::ifstream* file, int* label, char* buffer) {
- char label_char;
- if (FLAGS_is_cifar100) {
- // Skip the coarse label.
- file->read(&label_char, 1);
- }
- file->read(&label_char, 1);
- *label = label_char;
- // Yes, there are better ways to do it, like in-place swap... but I am too
- // lazy so let's just write it in a memory-wasteful way.
- std::array<char, kCIFARImageNBytes> channel_first_storage;
- file->read(channel_first_storage.data(), kCIFARImageNBytes);
- for (int c = 0; c < 3; ++c) {
- for (int i = 0; i < kCIFARSize * kCIFARSize; ++i) {
- buffer[i * 3 + c] =
- channel_first_storage[c * kCIFARSize * kCIFARSize + i];
- }
- }
- return;
-}
-
-void WriteToDB(const string& filename, const int num_items,
- const int& offset, db::DB* db) {
- TensorProtos protos;
- TensorProto* data = protos.add_protos();
- TensorProto* label = protos.add_protos();
- data->set_data_type(TensorProto::BYTE);
- data->add_dims(kCIFARSize);
- data->add_dims(kCIFARSize);
- data->add_dims(3);
- label->set_data_type(TensorProto::INT32);
- label->add_dims(1);
- label->add_int32_data(0);
-
- LOG(INFO) << "Converting file " << filename;
- std::ifstream data_file(filename.c_str(),
- std::ios::in | std::ios::binary);
- CAFFE_ENFORCE(data_file, "Unable to open file ", filename);
- char str_buffer[kCIFARImageNBytes];
- int label_value;
- std::unique_ptr<db::Transaction> transaction(db->NewTransaction());
- for (int itemid = 0; itemid < num_items; ++itemid) {
- ReadImage(&data_file, &label_value, str_buffer);
- data->set_byte_data(str_buffer, kCIFARImageNBytes);
- label->set_int32_data(0, label_value);
- snprintf(str_buffer, kCIFARImageNBytes, "%05d",
- offset + itemid);
- transaction->Put(string(str_buffer), protos.SerializeAsString());
- }
-}
-
-void ConvertCIFAR() {
- std::unique_ptr<db::DB> train_db(
- db::CreateDB(FLAGS_db, FLAGS_output_train_db_name, db::NEW));
- std::unique_ptr<db::DB> test_db(
- db::CreateDB(FLAGS_db, FLAGS_output_test_db_name, db::NEW));
-
- if (!FLAGS_is_cifar100) {
- // This is cifar 10.
- for (int fileid = 0; fileid < kCIFAR10TrainBatches; ++fileid) {
- stringstream train_file;
- train_file << FLAGS_input_folder << "/data_batch_" << fileid + 1
- << ".bin";
- WriteToDB(train_file.str(), kCIFAR10BatchSize,
- fileid * kCIFAR10BatchSize, train_db.get());
- }
- stringstream test_file;
- test_file << FLAGS_input_folder << "/test_batch.bin";
- WriteToDB(test_file.str(), kCIFAR10TestDataSize, 0, test_db.get());
- } else {
- // This is cifar 100.
- stringstream train_file;
- train_file << FLAGS_input_folder << "/train.bin";
- WriteToDB(train_file.str(), kCIFAR100TrainDataSize, 0, train_db.get());
- stringstream test_file;
- test_file << FLAGS_input_folder << "/test.bin";
- WriteToDB(test_file.str(), kCIFAR100TestDataSize, 0, test_db.get());
- }
-}
-
-} // namespace caffe2
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- caffe2::ConvertCIFAR();
- return 0;
-}
diff --git a/binaries/make_image_db.cc b/binaries/make_image_db.cc
deleted file mode 100644
index 3bbe15a..0000000
--- a/binaries/make_image_db.cc
+++ /dev/null
@@ -1,279 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This script converts an image dataset to a database.
-//
-// FLAGS_input_folder is the root folder that holds all the images
-//
-// FLAGS_list_file is the path to a file containing a list of files
-// and their labels, as follows:
-//
-// subfolder1/file1.JPEG 7
-// subfolder1/file2.JPEG 7
-// subfolder2/file1.JPEG 8
-// ...
-//
-
-#include <opencv2/opencv.hpp>
-
-#include <algorithm>
-#include <fstream>
-#include <queue>
-#include <random>
-#include <string>
-#include <thread>
-
-#include "caffe2/core/common.h"
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/core/logging.h"
-
-C10_DEFINE_bool(
- shuffle,
- false,
- "Randomly shuffle the order of images and their labels");
-C10_DEFINE_string(input_folder, "", "The input image file name.");
-C10_DEFINE_string(
- list_file,
- "",
- "The text file containing the list of images.");
-C10_DEFINE_string(output_db_name, "", "The output training leveldb name.");
-C10_DEFINE_string(db, "leveldb", "The db type.");
-C10_DEFINE_bool(
- raw,
- false,
- "If set, we pre-read the images and store the raw buffer.");
-C10_DEFINE_bool(color, true, "If set, load images in color.");
-C10_DEFINE_int(
- scale,
- 256,
- "If FLAGS_raw is set, scale the shorter edge to the given value.");
-C10_DEFINE_bool(warp, false, "If warp is set, warp the images to square.");
-C10_DEFINE_int(
- num_threads,
- -1,
- "Number of image parsing and conversion threads.");
-
-namespace caffe2 {
-
-class Converter {
- public:
- explicit Converter() {
- data_ = protos_.add_protos();
- label_ = protos_.add_protos();
- if (FLAGS_raw) {
- data_->set_data_type(TensorProto::BYTE);
- data_->add_dims(0);
- data_->add_dims(0);
- if (FLAGS_color) {
- data_->add_dims(3);
- }
- } else {
- data_->set_data_type(TensorProto::STRING);
- data_->add_dims(1);
- data_->add_string_data("");
- }
- label_->set_data_type(TensorProto::INT32);
- label_->add_dims(1);
- label_->add_int32_data(0);
- }
-
- ~Converter() {
- if (thread_.joinable()) {
- thread_.join();
- }
- }
-
- void queue(const std::pair<std::string, int>& pair) {
- in_.push(pair);
- }
-
- void start() {
- thread_ = std::thread(&Converter::run, this);
- }
-
- std::string get() {
- std::unique_lock<std::mutex> lock(mutex_);
- while (out_.empty()) {
- cv_.wait(lock);
- }
-
- auto value = out_.front();
- out_.pop();
- cv_.notify_one();
- return value;
- }
-
- void run() {
- const auto& input_folder = FLAGS_input_folder;
- std::unique_lock<std::mutex> lock(mutex_);
- std::string value;
- while (!in_.empty()) {
- auto pair = in_.front();
- in_.pop();
- lock.unlock();
-
- label_->set_int32_data(0, pair.second);
-
- // Add raw file contents to DB if !raw
- if (!FLAGS_raw) {
- std::ifstream image_file_stream(input_folder + pair.first);
- if (!image_file_stream) {
- LOG(ERROR) << "Cannot open " << input_folder << pair.first
- << ". Skipping.";
- } else {
- data_->mutable_string_data(0)->assign(
- std::istreambuf_iterator<char>(image_file_stream),
- std::istreambuf_iterator<char>());
- }
- } else {
- // Load image
- cv::Mat img = cv::imread(
- input_folder + pair.first,
- FLAGS_color ? cv::IMREAD_COLOR : cv::IMREAD_GRAYSCALE);
-
- // Resize image
- cv::Mat resized_img;
- int scaled_width, scaled_height;
- if (FLAGS_warp) {
- scaled_width = FLAGS_scale;
- scaled_height = FLAGS_scale;
- } else if (img.rows > img.cols) {
- scaled_width = FLAGS_scale;
- scaled_height = static_cast<float>(img.rows) * FLAGS_scale / img.cols;
- } else {
- scaled_height = FLAGS_scale;
- scaled_width = static_cast<float>(img.cols) * FLAGS_scale / img.rows;
- }
- cv::resize(
- img,
- resized_img,
- cv::Size(scaled_width, scaled_height),
- 0,
- 0,
- cv::INTER_LINEAR);
- data_->set_dims(0, scaled_height);
- data_->set_dims(1, scaled_width);
-
- // Assert we don't have to deal with alignment
- DCHECK(resized_img.isContinuous());
- auto nbytes = resized_img.total() * resized_img.elemSize();
- data_->set_byte_data(resized_img.ptr(), nbytes);
- }
-
- protos_.SerializeToString(&value);
-
- // Add serialized proto to out queue or wait if it is not empty
- lock.lock();
- while (!out_.empty()) {
- cv_.wait(lock);
- }
- out_.push(value);
- cv_.notify_one();
- }
- }
-
- protected:
- TensorProtos protos_;
- TensorProto* data_;
- TensorProto* label_;
- std::queue<std::pair<std::string, int>> in_;
- std::queue<std::string> out_;
-
- std::mutex mutex_;
- std::condition_variable cv_;
- std::thread thread_;
-};
-
-void ConvertImageDataset(
- const string& input_folder,
- const string& list_filename,
- const string& output_db_name,
- const bool /*shuffle*/) {
- std::ifstream list_file(list_filename);
- std::vector<std::pair<std::string, int> > lines;
- std::string filename;
- int file_label;
- while (list_file >> filename >> file_label) {
- lines.push_back(std::make_pair(filename, file_label));
- }
-
- if (FLAGS_shuffle) {
- LOG(INFO) << "Shuffling data";
- std::shuffle(lines.begin(), lines.end(), std::default_random_engine(1701));
- }
-
- auto num_threads = FLAGS_num_threads;
- if (num_threads < 1) {
- num_threads = std::thread::hardware_concurrency();
- }
-
- LOG(INFO) << "Processing " << lines.size() << " images...";
- LOG(INFO) << "Opening DB " << output_db_name;
-
- auto db = db::CreateDB(FLAGS_db, output_db_name, db::NEW);
- auto transaction = db->NewTransaction();
-
- LOG(INFO) << "Using " << num_threads << " processing threads...";
- std::vector<Converter> converters(num_threads);
-
- // Queue entries across converters
- for (auto i = 0; i < lines.size(); i++) {
- converters[i % converters.size()].queue(lines[i]);
- }
-
- // Start all converters
- for (auto& converter : converters) {
- converter.start();
- }
-
- constexpr auto key_max_length = 256;
- char key_cstr[key_max_length];
- int count = 0;
- for (auto i = 0; i < lines.size(); i++) {
- // Get serialized proto for this entry
- auto value = converters[i % converters.size()].get();
-
- // Synthesize key for this entry
- auto key_len = snprintf(
- key_cstr, sizeof(key_cstr), "%08d_%s", i, lines[i].first.c_str());
- TORCH_DCHECK_LE(key_len, sizeof(key_cstr));
-
- // Put in db
- transaction->Put(string(key_cstr), std::move(value));
-
- if (++count % 1000 == 0) {
- // Commit the current writes.
- transaction->Commit();
- LOG(INFO) << "Processed " << count << " files.";
- }
- }
-
- // Commit final transaction
- transaction->Commit();
- LOG(INFO) << "Processed " << count << " files.";
-}
-
-} // namespace caffe2
-
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- caffe2::ConvertImageDataset(
- FLAGS_input_folder, FLAGS_list_file, FLAGS_output_db_name, FLAGS_shuffle);
- return 0;
-}
diff --git a/binaries/make_mnist_db.cc b/binaries/make_mnist_db.cc
deleted file mode 100644
index 53e9d87..0000000
--- a/binaries/make_mnist_db.cc
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This script converts the MNIST dataset to leveldb.
-// The MNIST dataset could be downloaded at
-// http://yann.lecun.com/exdb/mnist/
-
-#include <fstream> // NOLINT(readability/streams)
-#include <string>
-
-#include "caffe2/core/common.h"
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/core/logging.h"
-
-C10_DEFINE_string(image_file, "", "The input image file name.");
-C10_DEFINE_string(label_file, "", "The label file name.");
-C10_DEFINE_string(output_file, "", "The output db name.");
-C10_DEFINE_string(db, "leveldb", "The db type.");
-C10_DEFINE_int(
- data_limit,
- -1,
- "If set, only output this number of data points.");
-C10_DEFINE_bool(
- channel_first,
- false,
- "If set, write the data as channel-first (CHW order) as the old "
- "Caffe does.");
-
-namespace caffe2 {
-uint32_t swap_endian(uint32_t val) {
- val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
- return (val << 16) | (val >> 16);
-}
-
-void convert_dataset(const char* image_filename, const char* label_filename,
- const char* db_path, const int data_limit) {
- // Open files
- std::ifstream image_file(image_filename, std::ios::in | std::ios::binary);
- std::ifstream label_file(label_filename, std::ios::in | std::ios::binary);
- CAFFE_ENFORCE(image_file, "Unable to open file ", image_filename);
- CAFFE_ENFORCE(label_file, "Unable to open file ", label_filename);
- // Read the magic and the meta data
- uint32_t magic;
- uint32_t num_items;
- uint32_t num_labels;
- uint32_t rows;
- uint32_t cols;
-
- image_file.read(reinterpret_cast<char*>(&magic), 4);
- magic = swap_endian(magic);
- if (magic == 529205256) {
- LOG(FATAL) <<
- "It seems that you forgot to unzip the mnist dataset. You should "
- "first unzip them using e.g. gunzip on Linux.";
- }
- CAFFE_ENFORCE_EQ(magic, 2051, "Incorrect image file magic.");
- label_file.read(reinterpret_cast<char*>(&magic), 4);
- magic = swap_endian(magic);
- CAFFE_ENFORCE_EQ(magic, 2049, "Incorrect label file magic.");
- image_file.read(reinterpret_cast<char*>(&num_items), 4);
- num_items = swap_endian(num_items);
- label_file.read(reinterpret_cast<char*>(&num_labels), 4);
- num_labels = swap_endian(num_labels);
- CAFFE_ENFORCE_EQ(num_items, num_labels);
- image_file.read(reinterpret_cast<char*>(&rows), 4);
- rows = swap_endian(rows);
- image_file.read(reinterpret_cast<char*>(&cols), 4);
- cols = swap_endian(cols);
-
- // leveldb
- std::unique_ptr<db::DB> mnist_db(db::CreateDB(FLAGS_db, db_path, db::NEW));
- std::unique_ptr<db::Transaction> transaction(mnist_db->NewTransaction());
- // Storing to db
- char label_value;
- std::vector<char> pixels(rows * cols);
- int count = 0;
- const int kMaxKeyLength = 11;
- char key_cstr[kMaxKeyLength];
-
- TensorProtos protos;
- TensorProto* data = protos.add_protos();
- TensorProto* label = protos.add_protos();
- data->set_data_type(TensorProto::BYTE);
- if (FLAGS_channel_first) {
- data->add_dims(1);
- data->add_dims(rows);
- data->add_dims(cols);
- } else {
- data->add_dims(rows);
- data->add_dims(cols);
- data->add_dims(1);
- }
- label->set_data_type(TensorProto::INT32);
- label->add_int32_data(0);
-
- LOG(INFO) << "A total of " << num_items << " items.";
- LOG(INFO) << "Rows: " << rows << " Cols: " << cols;
- for (int item_id = 0; item_id < num_items; ++item_id) {
- image_file.read(pixels.data(), rows * cols);
- label_file.read(&label_value, 1);
- for (int i = 0; i < rows * cols; ++i) {
- data->set_byte_data(pixels.data(), rows * cols);
- }
- label->set_int32_data(0, static_cast<int>(label_value));
- snprintf(key_cstr, kMaxKeyLength, "%08d", item_id);
- string keystr(key_cstr);
-
- // Put in db
- transaction->Put(keystr, protos.SerializeAsString());
- if (++count % 1000 == 0) {
- transaction->Commit();
- }
- if (data_limit > 0 && count == data_limit) {
- LOG(INFO) << "Reached data limit of " << data_limit << ", stop.";
- break;
- }
- }
-}
-} // namespace caffe2
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- caffe2::convert_dataset(
- FLAGS_image_file.c_str(),
- FLAGS_label_file.c_str(),
- FLAGS_output_file.c_str(),
- FLAGS_data_limit);
- return 0;
-}
diff --git a/binaries/predictor_verifier.cc b/binaries/predictor_verifier.cc
deleted file mode 100644
index 47332dd..0000000
--- a/binaries/predictor_verifier.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "caffe2/core/flags.h"
-#include "caffe2/core/init.h"
-#include "caffe2/predictor/predictor.h"
-#include "caffe2/utils/proto_utils.h"
-
-C10_DEFINE_string(init_net, "", "The given path to the init protobuffer.");
-C10_DEFINE_string(
- predict_net,
- "",
- "The given path to the predict protobuffer.");
-
-namespace caffe2 {
-
-void run() {
- if (FLAGS_init_net.empty()) {
- LOG(FATAL) << "No init net specified. Use --init_net=/path/to/net.";
- }
- if (FLAGS_predict_net.empty()) {
- LOG(FATAL) << "No predict net specified. Use --predict_net=/path/to/net.";
- }
- caffe2::NetDef init_net, predict_net;
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &init_net));
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_predict_net, &predict_net));
- // Can be large due to constant fills
- VLOG(1) << "Init net: " << ProtoDebugString(init_net);
- LOG(INFO) << "Predict net: " << ProtoDebugString(predict_net);
- auto predictor = std::make_unique<Predictor>(init_net, predict_net);
- LOG(INFO) << "Checking that a null forward-pass works";
- Predictor::TensorList inputVec, outputVec;
- (*predictor)(inputVec, &outputVec);
- CAFFE_ENFORCE_GT(outputVec.size(), 0);
-}
-}
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- caffe2::run();
- // This is to allow us to use memory leak checks.
- caffe2::ShutdownProtobufLibrary();
- return 0;
-}
diff --git a/binaries/print_registered_core_operators.cc b/binaries/print_registered_core_operators.cc
deleted file mode 100644
index a96af0d..0000000
--- a/binaries/print_registered_core_operators.cc
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <iostream>
-#include <string>
-
-#include "caffe2/core/init.h"
-#include "caffe2/core/operator.h"
-#include "caffe2/core/operator_schema.h"
-
-C10_DEFINE_string(schema, "", "Print doc and schema of a particular operator");
-
-static bool HasSchema(const std::string& str) {
- return caffe2::OpSchemaRegistry::Schema(str);
-}
-
-static bool HasDoc(const std::string& str) {
- const auto* schema = caffe2::OpSchemaRegistry::Schema(str);
- return (schema != nullptr) && (schema->doc() != nullptr);
-}
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
-
- if (!FLAGS_schema.empty()) {
- const auto* schema = caffe2::OpSchemaRegistry::Schema(FLAGS_schema);
- if (!schema) {
- std::cerr << "Operator " << FLAGS_schema << " doesn't have a schema"
- << std::endl;
- return 1;
- }
- std::cout << "Operator " << FLAGS_schema << ": " << std::endl << *schema;
- return 0;
- }
-
- for (const auto& pair : *caffe2::gDeviceTypeRegistry()) {
- std::cout << "Device type " << pair.first
-#ifndef CAFFE2_USE_LITE_PROTO
- << " ("
- << at::DeviceTypeName(static_cast<caffe2::DeviceType>(pair.first))
- << ")"
-#endif
- << std::endl;
- for (const auto& key : pair.second->Keys()) {
- std::cout << "\t(schema: " << HasSchema(key) << ", doc: " << HasDoc(key)
- << ")\t" << key << std::endl;
- }
- }
-
- std::cout << "Operators that have gradients registered:" << std::endl;
- for (const auto& key : caffe2::GradientRegistry()->Keys()) {
- std::cout << "\t(schema: " << HasSchema(key) << ", doc: "
- << HasDoc(key) << ")\t"
- << key << std::endl;
- }
- return 0;
-}
diff --git a/binaries/run_plan.cc b/binaries/run_plan.cc
deleted file mode 100644
index c5d467b..0000000
--- a/binaries/run_plan.cc
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "caffe2/core/init.h"
-#include "caffe2/core/operator.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/utils/proto_utils.h"
-#include "caffe2/core/logging.h"
-
-C10_DEFINE_string(plan, "", "The given path to the plan protobuffer.");
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- if (FLAGS_plan.size() == 0) {
- LOG(ERROR) << "No plan specified. Use --plan=/path/to/plan.";
- return 0;
- }
- LOG(INFO) << "Loading plan: " << FLAGS_plan;
- caffe2::PlanDef plan_def;
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_plan, &plan_def));
- std::unique_ptr<caffe2::Workspace> workspace(new caffe2::Workspace());
- workspace->RunPlan(plan_def);
-
- // This is to allow us to use memory leak checks.
- caffe2::ShutdownProtobufLibrary();
- return 0;
-}
diff --git a/binaries/run_plan_mpi.cc b/binaries/run_plan_mpi.cc
deleted file mode 100644
index 476aec1..0000000
--- a/binaries/run_plan_mpi.cc
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <mpi.h>
-
-#include "c10/util/Flags.h"
-#include "caffe2/core/init.h"
-#include "caffe2/core/logging.h"
-#include "caffe2/core/operator.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/utils/proto_utils.h"
-
-C10_DEFINE_string(plan, "", "The given path to the plan protobuffer.");
-
-int main(int argc, char** argv) {
- c10::SetUsageMessage("Runs a caffe2 plan that has MPI operators in it.");
- int mpi_ret;
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_ret);
- if (mpi_ret != MPI_THREAD_MULTIPLE &&
- mpi_ret != MPI_THREAD_SERIALIZED) {
- std::cerr << "Caffe2 MPI requires the underlying MPI to support the "
- "MPI_THREAD_SERIALIZED or MPI_THREAD_MULTIPLE mode.\n";
- return 1;
- }
- caffe2::GlobalInit(&argc, &argv);
- LOG(INFO) << "Loading plan: " << FLAGS_plan;
- caffe2::PlanDef plan_def;
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_plan, &plan_def));
- std::unique_ptr<caffe2::Workspace> workspace(new caffe2::Workspace());
- workspace->RunPlan(plan_def);
-
- // This is to allow us to use memory leak checks.
- caffe2::ShutdownProtobufLibrary();
- MPI_Finalize();
- return 0;
-}
diff --git a/binaries/speed_benchmark.cc b/binaries/speed_benchmark.cc
deleted file mode 100644
index 6028cef..0000000
--- a/binaries/speed_benchmark.cc
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string>
-
-#include "caffe2/core/blob_serialization.h"
-#include "caffe2/core/init.h"
-#include "caffe2/core/logging.h"
-#include "caffe2/core/operator.h"
-#include "caffe2/core/tensor_int8.h"
-#ifdef CAFFE2_OPTIMIZER
-#include "caffe2/opt/optimizer.h"
-#endif
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/utils/proto_utils.h"
-#include "caffe2/utils/string_utils.h"
-
-C10_DEFINE_string(net, "", "The given net to benchmark.");
-C10_DEFINE_string(init_net, "", "The given net to initialize any parameters.");
-C10_DEFINE_string(
- input,
- "",
- "Input that is needed for running the network. If "
- "multiple input needed, use comma separated string.");
-C10_DEFINE_string(
- input_file,
- "",
- "Input file that contain the serialized protobuf for "
- "the input blobs. If multiple input needed, use comma "
- "separated string. Must have the same number of items "
- "as input does.");
-C10_DEFINE_string(
- input_dims,
- "",
- "Alternate to input_files, if all inputs are simple "
- "float TensorCPUs, specify the dimension using comma "
- "separated numbers. If multiple input needed, use "
- "semicolon to separate the dimension of different "
- "tensors.");
-C10_DEFINE_string(input_type, "", "Input type (uint8_t/float)");
-C10_DEFINE_string(
- output,
- "",
- "Output that should be dumped after the execution "
- "finishes. If multiple outputs are needed, use comma "
- "separated string. If you want to dump everything, pass "
- "'*' as the output value.");
-C10_DEFINE_string(
- output_folder,
- "",
- "The folder that the output should be written to. This "
- "folder must already exist in the file system.");
-C10_DEFINE_int(warmup, 0, "The number of iterations to warm up.");
-C10_DEFINE_int(iter, 10, "The number of iterations to run.");
-C10_DEFINE_int(opt, 0, "The level of optimization to run automatically.");
-C10_DEFINE_bool(
- run_individual,
- false,
- "Whether to benchmark individual operators.");
-
-C10_DEFINE_bool(force_engine, false, "Force engine field for all operators");
-C10_DEFINE_string(engine, "", "Forced engine field value");
-C10_DEFINE_bool(force_algo, false, "Force algo arg for all operators");
-C10_DEFINE_string(algo, "", "Forced algo arg value");
-
-using std::string;
-using std::unique_ptr;
-using std::vector;
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- unique_ptr<caffe2::Workspace> workspace(new caffe2::Workspace());
-
- // Run initialization network.
- caffe2::NetDef net_def;
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_init_net, &net_def));
- CAFFE_ENFORCE(workspace->RunNetOnce(net_def));
-
- // Load input.
- if (FLAGS_input.size()) {
- vector<string> input_names = caffe2::split(',', FLAGS_input);
- if (FLAGS_input_file.size()) {
- vector<string> input_files = caffe2::split(',', FLAGS_input_file);
- CAFFE_ENFORCE_EQ(
- input_names.size(),
- input_files.size(),
- "Input name and file should have the same number.");
- for (int i = 0; i < input_names.size(); ++i) {
- caffe2::BlobProto blob_proto;
- CAFFE_ENFORCE(caffe2::ReadProtoFromFile(input_files[i], &blob_proto));
- DeserializeBlob(blob_proto, workspace->CreateBlob(input_names[i]));
- }
- } else if (FLAGS_input_dims.size() || FLAGS_input_type.size()) {
- CAFFE_ENFORCE_GE(
- FLAGS_input_dims.size(),
- 0,
- "Input dims must be specified when input tensors are used.");
- CAFFE_ENFORCE_GE(
- FLAGS_input_type.size(),
- 0,
- "Input type must be specified when input tensors are used.");
-
- vector<string> input_dims_list = caffe2::split(';', FLAGS_input_dims);
- CAFFE_ENFORCE_EQ(
- input_names.size(),
- input_dims_list.size(),
- "Input name and dims should have the same number of items.");
- vector<string> input_type_list = caffe2::split(';', FLAGS_input_type);
- CAFFE_ENFORCE_EQ(
- input_names.size(),
- input_type_list.size(),
- "Input name and type should have the same number of items.");
- for (size_t i = 0; i < input_names.size(); ++i) {
- vector<string> input_dims_str = caffe2::split(',', input_dims_list[i]);
- vector<int> input_dims;
- for (const string& s : input_dims_str) {
- input_dims.push_back(std::stoi(s));
- }
- caffe2::Blob* blob = workspace->GetBlob(input_names[i]);
- if (blob == nullptr) {
- blob = workspace->CreateBlob(input_names[i]);
- }
- if (input_type_list[i] == "uint8_t") {
- caffe2::int8::Int8TensorCPU* tensor =
- blob->GetMutable<caffe2::int8::Int8TensorCPU>();
- TORCH_CHECK_NOTNULL(tensor);
- tensor->t.Resize(input_dims);
- tensor->t.mutable_data<uint8_t>();
- } else if (input_type_list[i] == "float") {
- caffe2::TensorCPU* tensor = BlobGetMutableTensor(blob, caffe2::CPU);
- TORCH_CHECK_NOTNULL(tensor);
- tensor->Resize(input_dims);
- tensor->mutable_data<float>();
- } else {
- CAFFE_THROW("Unsupported input type: ", input_type_list[i]);
- }
- }
- } else {
- CAFFE_THROW(
- "You requested input tensors, but neither input_file nor "
- "input_dims is set.");
- }
- }
-
- // Run main network.
- CAFFE_ENFORCE(ReadProtoFromFile(FLAGS_net, &net_def));
- if (!net_def.has_name()) {
- net_def.set_name("benchmark");
- }
- // force changing engine and algo
- if (FLAGS_force_engine) {
- LOG(INFO) << "force engine be: " << FLAGS_engine;
- for (const auto& op : net_def.op()) {
- const_cast<caffe2::OperatorDef*>(&op)->set_engine(FLAGS_engine);
- }
- }
- if (FLAGS_force_algo) {
- LOG(INFO) << "force algo be: " << FLAGS_algo;
- for (const auto& op : net_def.op()) {
- caffe2::GetMutableArgument(
- "algo", true, const_cast<caffe2::OperatorDef*>(&op))
- ->set_s(FLAGS_algo);
- }
- }
- if (FLAGS_opt) {
-#ifdef CAFFE2_OPTIMIZER
- net_def = caffe2::opt::optimize(net_def, workspace.get(), FLAGS_opt);
-#else
- LOG(WARNING) << "Caffe2 not compiled with optimization passes.";
-#endif
- }
-
- caffe2::NetBase* net = workspace->CreateNet(net_def);
- TORCH_CHECK_NOTNULL(net);
- CAFFE_ENFORCE(net->Run());
- net->TEST_Benchmark(FLAGS_warmup, FLAGS_iter, FLAGS_run_individual);
-
- string output_prefix =
- FLAGS_output_folder.size() ? FLAGS_output_folder + "/" : "";
- if (FLAGS_output.size()) {
- vector<string> output_names = caffe2::split(',', FLAGS_output);
- if (FLAGS_output == "*") {
- output_names = workspace->Blobs();
- }
- for (const string& name : output_names) {
- CAFFE_ENFORCE(
- workspace->HasBlob(name),
- "You requested a non-existing blob: ",
- name);
- string serialized = SerializeBlob(*workspace->GetBlob(name), name);
- string output_filename = output_prefix + name;
- caffe2::WriteStringToFile(serialized, output_filename.c_str());
- }
- }
-
- return 0;
-}
diff --git a/binaries/split_db.cc b/binaries/split_db.cc
deleted file mode 100644
index 34c0b60..0000000
--- a/binaries/split_db.cc
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <string>
-#include <sstream>
-
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/proto/caffe2_pb.h"
-#include "caffe2/core/logging.h"
-
-C10_DEFINE_string(input_db, "", "The input db.");
-C10_DEFINE_int(splits, 0, "The number of splits.");
-C10_DEFINE_string(db_type, "", "The db type.");
-C10_DEFINE_int(batch_size, 1000, "The write batch size.");
-
-namespace caffe2 {
-
-static int Split(int argc, char** argv) {
- GlobalInit(&argc, &argv);
-
- CAFFE_ENFORCE(FLAGS_input_db.size(), "Must specify --input_db=/path/to/db.");
- CAFFE_ENFORCE(FLAGS_splits > 0, "Must specify a nonnegative split number.");
- CAFFE_ENFORCE(FLAGS_db_type.size(), "Must specify --db_type=[a db type].");
-
- unique_ptr<db::DB> in_db(
- db::CreateDB(FLAGS_db_type, FLAGS_input_db, db::READ));
- CAFFE_ENFORCE(in_db != nullptr, "Cannot open input db: ", FLAGS_input_db);
- unique_ptr<db::Cursor> cursor(in_db->NewCursor());
- // This usually won't happen, but FWIW.
- CAFFE_ENFORCE(
- cursor != nullptr, "Cannot obtain cursor for input db: ", FLAGS_input_db);
-
- vector<unique_ptr<db::DB>> out_dbs;
- vector<unique_ptr<db::Transaction>> transactions;
- for (int i = 0; i < FLAGS_splits; ++i) {
- out_dbs.push_back(unique_ptr<db::DB>(db::CreateDB(
- FLAGS_db_type, FLAGS_input_db + "_split_" + to_string(i), db::NEW)));
- CAFFE_ENFORCE(out_dbs.back().get(), "Cannot create output db #", i);
- transactions.push_back(
- unique_ptr<db::Transaction>(out_dbs[i]->NewTransaction()));
- CAFFE_ENFORCE(
- transactions.back().get(), "Cannot get transaction for output db #", i);
- }
-
- int count = 0;
- for (; cursor->Valid(); cursor->Next()) {
- transactions[count % FLAGS_splits]->Put(cursor->key(), cursor->value());
- if (++count % FLAGS_batch_size == 0) {
- for (int i = 0; i < FLAGS_splits; ++i) {
- transactions[i]->Commit();
- }
- LOG(INFO) << "Split " << count << " items so far.";
- }
- }
- LOG(INFO) << "A total of " << count << " items processed.";
- return 0;
-}
-
-} // namespace caffe2
-
-int main(int argc, char** argv) {
- return caffe2::Split(argc, argv);
-}
diff --git a/binaries/tutorial_blob.cc b/binaries/tutorial_blob.cc
deleted file mode 100644
index ac74ebb..0000000
--- a/binaries/tutorial_blob.cc
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "caffe2/core/blob.h"
-#include "caffe2/core/init.h"
-#include "caffe2/core/tensor.h"
-#include "caffe2/core/logging.h"
-
-// We will be lazy and just use the whole namespace.
-using namespace caffe2;
-
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
- caffe2::ShowLogInfoToStderr();
-
- LOG(INFO) <<
- "This script corresponds to the Blob part of the Caffe2 C++ "
- "tutorial.";
-
- LOG(INFO) << "Let's create a blob myblob.";
-
- Blob myblob;
-
- LOG(INFO) << "Let's set it to int and set the value to 10.";
-
- int* myint = myblob.GetMutable<int>();
- *myint = 10;
-
- LOG(INFO)
- << "Is the blob type int? "
- << myblob.IsType<int>();
-
- LOG(INFO)
- << "Is the blob type float? "
- << myblob.IsType<float>();
-
- const int& myint_const = myblob.Get<int>();
- LOG(INFO)
- << "The value of the int number stored in the blob is: "
- << myint_const;
-
- LOG(INFO)
- << "Let's try to get a float pointer. This will trigger an exception.";
-
- try {
- const float& myfloat = myblob.Get<float>();
- LOG(FATAL) << "This line should never happen.";
- } catch (std::exception& e) {
- LOG(INFO)
- << "As expected, we got an exception. Its content says: "
- << e.what();
- }
-
- LOG(INFO) <<
- "However, we can change the content type (and destroy the old "
- "content) by calling GetMutable. Let's change it to double.";
-
- double* mydouble = myblob.GetMutable<double>();
- *mydouble = 3.14;
-
- LOG(INFO) << "The new content is: " << myblob.Get<double>();
-
- LOG(INFO) <<
- "If we have a pre-created object, we can use Reset() to transfer the "
- "object to a blob.";
-
- std::string* pvec = new std::string();
- myblob.Reset(pvec); // no need to release pvec, myblob takes ownership.
-
- LOG(INFO) << "Is the blob now of type string? "
- << myblob.IsType<std::string>();
-
- LOG(INFO) << "This concludes the blob tutorial.";
- return 0;
-}
diff --git a/binaries/zmq_feeder.cc b/binaries/zmq_feeder.cc
deleted file mode 100644
index 8a6a71f..0000000
--- a/binaries/zmq_feeder.cc
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This binary provides an easy way to open a zeromq server and feeds data to
-// clients connect to it. It uses the Caffe2 db as the backend, thus allowing
-// one to convert any db-compliant storage to a zeromq service.
-
-#include "caffe2/core/db.h"
-#include "caffe2/core/init.h"
-#include "caffe2/core/logging.h"
-#include "caffe2/utils/zmq_helper.h"
-
-C10_DEFINE_string(server, "tcp://*:5555", "The server address.");
-C10_DEFINE_string(input_db, "", "The input db.");
-C10_DEFINE_string(input_db_type, "", "The input db type.");
-
-
-int main(int argc, char** argv) {
- caffe2::GlobalInit(&argc, &argv);
-
- LOG(INFO) << "Opening DB...";
- auto in_db = caffe2::db::CreateDB(
- FLAGS_input_db_type, FLAGS_input_db, caffe2::db::READ);
- CAFFE_ENFORCE(
- in_db,
- "Cannot load input db " + FLAGS_input_db + " of expected type " +
- FLAGS_input_db_type);
- auto cursor = in_db->NewCursor();
- LOG(INFO) << "DB opened.";
-
- LOG(INFO) << "Starting ZeroMQ server...";
-
- // Socket to talk to clients
- caffe2::ZmqSocket sender(ZMQ_PUSH);
- sender.Bind(FLAGS_server);
- LOG(INFO) << "Server created at " << FLAGS_server;
-
- while (1) {
- VLOG(1) << "Sending " << cursor->key();
- sender.SendTillSuccess(cursor->key(), ZMQ_SNDMORE);
- sender.SendTillSuccess(cursor->value(), 0);
- cursor->Next();
- if (!cursor->Valid()) {
- cursor->SeekToFirst();
- }
- }
- // We do not do an elegant quit since this binary is going to be terminated by
- // control+C.
- return 0;
-}
diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
index bcd171f..569bc01 100644
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -122,8 +122,6 @@
if(USE_NVRTC)
add_subdirectory(cuda_rtc)
endif()
- add_subdirectory(db)
- add_subdirectory(distributed)
add_subdirectory(ideep)
add_subdirectory(mobile)
add_subdirectory(mpi)
diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
index a8313f2..3000e2d 100644
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -865,18 +865,6 @@
caffe2_update_option(USE_FAKELOWP OFF)
endif()
-# ---[ LMDB
-if(USE_LMDB)
- find_package(LMDB)
- if(LMDB_FOUND)
- include_directories(SYSTEM ${LMDB_INCLUDE_DIR})
- list(APPEND Caffe2_DEPENDENCY_LIBS ${LMDB_LIBRARIES})
- else()
- message(WARNING "Not compiling with LMDB. Suppress this warning with -DUSE_LMDB=OFF")
- caffe2_update_option(USE_LMDB OFF)
- endif()
-endif()
-
if(USE_OPENCL)
message(INFO "USING OPENCL")
find_package(OpenCL REQUIRED)
@@ -884,22 +872,6 @@
list(APPEND Caffe2_DEPENDENCY_LIBS ${OpenCL_LIBRARIES})
endif()
-# ---[ LevelDB
-# ---[ Snappy
-if(USE_LEVELDB)
- find_package(LevelDB)
- find_package(Snappy)
- if(LEVELDB_FOUND AND SNAPPY_FOUND)
- include_directories(SYSTEM ${LevelDB_INCLUDE})
- list(APPEND Caffe2_DEPENDENCY_LIBS ${LevelDB_LIBRARIES})
- include_directories(SYSTEM ${Snappy_INCLUDE_DIR})
- list(APPEND Caffe2_DEPENDENCY_LIBS ${Snappy_LIBRARIES})
- else()
- message(WARNING "Not compiling with LevelDB. Suppress this warning with -DUSE_LEVELDB=OFF")
- caffe2_update_option(USE_LEVELDB OFF)
- endif()
-endif()
-
# ---[ NUMA
if(USE_NUMA)
if(LINUX)
@@ -914,30 +886,6 @@
endif()
endif()
-# ---[ ZMQ
-if(USE_ZMQ)
- find_package(ZMQ)
- if(ZMQ_FOUND)
- include_directories(SYSTEM ${ZMQ_INCLUDE_DIR})
- list(APPEND Caffe2_DEPENDENCY_LIBS ${ZMQ_LIBRARIES})
- else()
- message(WARNING "Not compiling with ZMQ. Suppress this warning with -DUSE_ZMQ=OFF")
- caffe2_update_option(USE_ZMQ OFF)
- endif()
-endif()
-
-# ---[ Redis
-if(USE_REDIS)
- find_package(Hiredis)
- if(HIREDIS_FOUND)
- include_directories(SYSTEM ${Hiredis_INCLUDE})
- list(APPEND Caffe2_DEPENDENCY_LIBS ${Hiredis_LIBRARIES})
- else()
- message(WARNING "Not compiling with Redis. Suppress this warning with -DUSE_REDIS=OFF")
- caffe2_update_option(USE_REDIS OFF)
- endif()
-endif()
-
if(USE_ITT)
find_package(ITT)
if(ITT_FOUND)
diff --git a/cmake/Modules/FindHiredis.cmake b/cmake/Modules/FindHiredis.cmake
deleted file mode 100644
index f1389de..0000000
--- a/cmake/Modules/FindHiredis.cmake
+++ /dev/null
@@ -1,23 +0,0 @@
-# Find the Hiredis libraries
-#
-# The following variables are optionally searched for defaults
-# HIREDIS_ROOT_DIR: Base directory where all Hiredis components are found
-#
-# The following are set after configuration is done:
-# HIREDIS_FOUND
-# Hiredis_INCLUDE_DIR
-# Hiredis_LIBRARIES
-
-find_path(Hiredis_INCLUDE_DIR NAMES hiredis/hiredis.h
- PATHS ${HIREDIS_ROOT_DIR} ${HIREDIS_ROOT_DIR}/include)
-
-find_library(Hiredis_LIBRARIES NAMES hiredis
- PATHS ${HIREDIS_ROOT_DIR} ${HIREDIS_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Hiredis DEFAULT_MSG Hiredis_INCLUDE_DIR Hiredis_LIBRARIES)
-
-if(HIREDIS_FOUND)
- message(STATUS "Found Hiredis (include: ${Hiredis_INCLUDE_DIR}, library: ${Hiredis_LIBRARIES})")
- mark_as_advanced(Hiredis_INCLUDE_DIR Hiredis_LIBRARIES)
-endif()
diff --git a/cmake/Modules/FindLMDB.cmake b/cmake/Modules/FindLMDB.cmake
deleted file mode 100644
index 2f0adb1..0000000
--- a/cmake/Modules/FindLMDB.cmake
+++ /dev/null
@@ -1,32 +0,0 @@
-# Try to find the LMBD libraries and headers
-# LMDB_FOUND - system has LMDB lib
-# LMDB_INCLUDE_DIR - the LMDB include directory
-# LMDB_LIBRARIES - Libraries needed to use LMDB
-
-# FindCWD based on FindGMP by:
-# Copyright (c) 2006, Laurent Montel, <montel@kde.org>
-#
-# Redistribution and use is allowed according to the terms of the BSD license.
-
-# Adapted from FindCWD by:
-# Copyright 2013 Conrad Steenberg <conrad.steenberg@gmail.com>
-# Aug 31, 2013
-
-if(MSVC)
- find_package(LMDB NO_MODULE)
-else()
- find_path(LMDB_INCLUDE_DIR NAMES lmdb.h PATHS "$ENV{LMDB_DIR}/include")
- find_library(LMDB_LIBRARIES NAMES lmdb PATHS "$ENV{LMDB_DIR}/lib" )
-endif()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(LMDB DEFAULT_MSG LMDB_INCLUDE_DIR LMDB_LIBRARIES)
-
-if(LMDB_FOUND)
- message(STATUS "Found lmdb (include: ${LMDB_INCLUDE_DIR}, library: ${LMDB_LIBRARIES})")
- mark_as_advanced(LMDB_INCLUDE_DIR LMDB_LIBRARIES)
-
- caffe_parse_header(${LMDB_INCLUDE_DIR}/lmdb.h
- LMDB_VERSION_LINES MDB_VERSION_MAJOR MDB_VERSION_MINOR MDB_VERSION_PATCH)
- set(LMDB_VERSION "${MDB_VERSION_MAJOR}.${MDB_VERSION_MINOR}.${MDB_VERSION_PATCH}")
-endif()
diff --git a/cmake/Modules/FindLevelDB.cmake b/cmake/Modules/FindLevelDB.cmake
deleted file mode 100644
index 320c246..0000000
--- a/cmake/Modules/FindLevelDB.cmake
+++ /dev/null
@@ -1,44 +0,0 @@
-# - Find LevelDB
-#
-# LevelDB_INCLUDES - List of LevelDB includes
-# LevelDB_LIBRARIES - List of libraries when using LevelDB.
-# LevelDB_FOUND - True if LevelDB found.
-
-# Look for the header file.
-find_path(LevelDB_INCLUDE NAMES leveldb/db.h
- PATHS $ENV{LEVELDB_ROOT}/include /opt/local/include /usr/local/include /usr/include
- DOC "Path in which the file leveldb/db.h is located." )
-
-# Look for the library.
-find_library(LevelDB_LIBRARY NAMES leveldb
- PATHS /usr/lib $ENV{LEVELDB_ROOT}/lib
- DOC "Path to leveldb library." )
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(LevelDB DEFAULT_MSG LevelDB_INCLUDE LevelDB_LIBRARY)
-
-if(LEVELDB_FOUND)
- message(STATUS "Found LevelDB (include: ${LevelDB_INCLUDE}, library: ${LevelDB_LIBRARY})")
- set(LevelDB_INCLUDES ${LevelDB_INCLUDE})
- set(LevelDB_LIBRARIES ${LevelDB_LIBRARY})
- mark_as_advanced(LevelDB_INCLUDE LevelDB_LIBRARY)
-
- if(EXISTS "${LevelDB_INCLUDE}/leveldb/db.h")
- file(STRINGS "${LevelDB_INCLUDE}/leveldb/db.h" __version_lines
- REGEX "static const int k[^V]+Version[ \t]+=[ \t]+[0-9]+;")
-
- foreach(__line ${__version_lines})
- if(__line MATCHES "[^k]+kMajorVersion[ \t]+=[ \t]+([0-9]+);")
- set(LEVELDB_VERSION_MAJOR ${CMAKE_MATCH_1})
- elseif(__line MATCHES "[^k]+kMinorVersion[ \t]+=[ \t]+([0-9]+);")
- set(LEVELDB_VERSION_MINOR ${CMAKE_MATCH_1})
- endif()
- endforeach()
-
- if(LEVELDB_VERSION_MAJOR AND LEVELDB_VERSION_MINOR)
- set(LEVELDB_VERSION "${LEVELDB_VERSION_MAJOR}.${LEVELDB_VERSION_MINOR}")
- endif()
-
- # caffe_clear_vars(__line __version_lines)
- endif()
-endif()
diff --git a/cmake/Modules/FindRocksDB.cmake b/cmake/Modules/FindRocksDB.cmake
deleted file mode 100644
index ef7f3b5..0000000
--- a/cmake/Modules/FindRocksDB.cmake
+++ /dev/null
@@ -1,23 +0,0 @@
-# Find the RocksDB libraries
-#
-# The following variables are optionally searched for defaults
-# ROCKSDB_ROOT_DIR: Base directory where all RocksDB components are found
-#
-# The following are set after configuration is done:
-# ROCKSDB_FOUND
-# RocksDB_INCLUDE_DIR
-# RocksDB_LIBRARIES
-
-find_path(RocksDB_INCLUDE_DIR NAMES rocksdb/db.h
- PATHS ${ROCKSDB_ROOT_DIR} ${ROCKSDB_ROOT_DIR}/include)
-
-find_library(RocksDB_LIBRARIES NAMES rocksdb
- PATHS ${ROCKSDB_ROOT_DIR} ${ROCKSDB_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(RocksDB DEFAULT_MSG RocksDB_INCLUDE_DIR RocksDB_LIBRARIES)
-
-if(ROCKSDB_FOUND)
- message(STATUS "Found RocksDB (include: ${RocksDB_INCLUDE_DIR}, library: ${RocksDB_LIBRARIES})")
- mark_as_advanced(RocksDB_INCLUDE_DIR RocksDB_LIBRARIES)
-endif()
diff --git a/cmake/Modules/FindSnappy.cmake b/cmake/Modules/FindSnappy.cmake
deleted file mode 100644
index 97219e8..0000000
--- a/cmake/Modules/FindSnappy.cmake
+++ /dev/null
@@ -1,26 +0,0 @@
-# Find the Snappy libraries
-#
-# The following variables are optionally searched for defaults
-# SNAPPY_ROOT_DIR: Base directory where all Snappy components are found
-#
-# The following are set after configuration is done:
-# SNAPPY_FOUND
-# Snappy_INCLUDE_DIR
-# Snappy_LIBRARIES
-
-find_path(Snappy_INCLUDE_DIR NAMES snappy.h
- PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/include)
-
-find_library(Snappy_LIBRARIES NAMES snappy
- PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_INCLUDE_DIR Snappy_LIBRARIES)
-
-if(SNAPPY_FOUND)
- message(STATUS "Found Snappy (include: ${Snappy_INCLUDE_DIR}, library: ${Snappy_LIBRARIES})")
- mark_as_advanced(Snappy_INCLUDE_DIR Snappy_LIBRARIES)
- caffe_parse_header(${Snappy_INCLUDE_DIR}/snappy-stubs-public.h
- SNAPPY_VERION_LINES SNAPPY_MAJOR SNAPPY_MINOR SNAPPY_PATCHLEVEL)
- set(Snappy_VERSION "${SNAPPY_MAJOR}.${SNAPPY_MINOR}.${SNAPPY_PATCHLEVEL}")
-endif()
diff --git a/cmake/Modules/FindZMQ.cmake b/cmake/Modules/FindZMQ.cmake
deleted file mode 100644
index 43ce50b..0000000
--- a/cmake/Modules/FindZMQ.cmake
+++ /dev/null
@@ -1,32 +0,0 @@
-# Find the ZMQ libraries
-#
-# The following variables are optionally searched for defaults
-# ZMQ_ROOT_DIR: Base directory where all ZMQ components are found
-#
-# The following are set after configuration is done:
-# ZMQ_FOUND
-# ZMQ_INCLUDE_DIR
-# ZMQ_LIBRARIES
-# ZMQ_VERSION_MAJOR
-
-find_path(ZMQ_INCLUDE_DIR NAMES zmq.h
- PATHS ${ZMQ_ROOT_DIR} ${ZMQ_ROOT_DIR}/include)
-
-find_library(ZMQ_LIBRARIES NAMES zmq
- PATHS ${ZMQ_ROOT_DIR} ${ZMQ_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(ZMQ DEFAULT_MSG ZMQ_INCLUDE_DIR ZMQ_LIBRARIES)
-
-if(ZMQ_FOUND)
- message(STATUS "Found ZMQ (include: ${ZMQ_INCLUDE_DIR}, library: ${ZMQ_LIBRARIES})")
- mark_as_advanced(ZMQ_INCLUDE_DIR ZMQ_LIBRARIES)
-
- caffe_parse_header(${ZMQ_INCLUDE_DIR}/zmq.h ZMQ_VERSION_LINES ZMQ_VERSION_MAJOR)
- if(${ZMQ_VERSION_MAJOR} VERSION_LESS "3")
- message(WARNING "Caffe2 requires zmq version 3 or above, but found " ${ZMQ_VERSION_MAJOR} ". Disabling zmq for now.")
- set(ZMQ_FOUND)
- else()
-
- endif()
-endif()
diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake
index 8a0ae2e..9b28599 100644
--- a/cmake/Summary.cmake
+++ b/cmake/Summary.cmake
@@ -130,16 +130,7 @@
message(STATUS " USE_KINETO : ${USE_KINETO}")
message(STATUS " USE_GFLAGS : ${USE_GFLAGS}")
message(STATUS " USE_GLOG : ${USE_GLOG}")
- message(STATUS " USE_LEVELDB : ${USE_LEVELDB}")
- if(${USE_LEVELDB})
- message(STATUS " LevelDB version : ${LEVELDB_VERSION}")
- message(STATUS " Snappy version : ${Snappy_VERSION}")
- endif()
message(STATUS " USE_LITE_PROTO : ${USE_LITE_PROTO}")
- message(STATUS " USE_LMDB : ${USE_LMDB}")
- if(${USE_LMDB})
- message(STATUS " LMDB version : ${LMDB_VERSION}")
- endif()
message(STATUS " USE_METAL : ${USE_METAL}")
message(STATUS " USE_PYTORCH_METAL : ${USE_PYTORCH_METAL}")
message(STATUS " USE_PYTORCH_METAL_EXPORT : ${USE_PYTORCH_METAL_EXPORT}")
@@ -178,9 +169,6 @@
message(STATUS " USE_QNNPACK : ${USE_QNNPACK}")
message(STATUS " USE_PYTORCH_QNNPACK : ${USE_PYTORCH_QNNPACK}")
message(STATUS " USE_XNNPACK : ${USE_XNNPACK}")
- message(STATUS " USE_REDIS : ${USE_REDIS}")
- message(STATUS " USE_ROCKSDB : ${USE_ROCKSDB}")
- message(STATUS " USE_ZMQ : ${USE_ZMQ}")
message(STATUS " USE_DISTRIBUTED : ${USE_DISTRIBUTED}")
if(${USE_DISTRIBUTED})
message(STATUS " USE_MPI : ${USE_MPI}")
diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt
index 9a5e727..598cac6 100644
--- a/modules/CMakeLists.txt
+++ b/modules/CMakeLists.txt
@@ -2,7 +2,6 @@
add_subdirectory(detectron)
add_subdirectory(module_test)
add_subdirectory(observers)
-add_subdirectory(rocksdb)
# Finally, set Caffe2_MODULES to parent scope.
set(Caffe2_MODULES ${Caffe2_MODULES} PARENT_SCOPE)
diff --git a/modules/rocksdb/CMakeLists.txt b/modules/rocksdb/CMakeLists.txt
deleted file mode 100644
index 7983ab9..0000000
--- a/modules/rocksdb/CMakeLists.txt
+++ /dev/null
@@ -1,78 +0,0 @@
-# ---[ RocksDB module
-# In addition to being a useful module itself, RocksDB is also an exemplar
-# case where show how one should built a Caffe2 module inside the Caffe2
-# repository.
-#
-# This cmake file achieves two build modes:
-# (1) If one is invoking the main Caffe2 build, we will check a USE_* option,
-# in this case USE_ROCKSDB, to test if we want to build this module.
-# (2) if we are building it in a standalone way, we will find the preinstalled
-# Caffe2 library, and then build the library and install it.
-
-# ---[ First, determine if we are building with the main repo or not.
-# This is guarded by the CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO variable. It then
-# routes build to two paths:
-# (1) When we are building with the main repo, the caffe2_library is going to
-# be already defined, and all related paths will be defined too. So we will
-# simply test if the main repo build wants to build this module, in our
-# case by the variable "USE_ROCKSDB".
-# (2) When we are not building with the main repo, we will need to do the usual
-# cmake setup: version checks, project options, find dependent packages,
-# etc.
-if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO)
- if(NOT USE_ROCKSDB)
- return()
- endif()
-else()
- cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
- project(caffe2_rocksdb CXX)
- find_package(Caffe2 REQUIRED)
- option(BUILD_SHARED_LIBS "Build shared libs." ON)
- list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../cmake/Modules)
-endif()
-
-
-# ---[ Second, find dependencies.
-# This one should be similar to the standard dependency discovery in normal
-# cmake. Note that for modules that are located in the Caffe2 repository,
-# cmake related files, such as FindRocksDB in this case, should live in the
-# cmake/ folder under root.
-find_package(RocksDB CONFIG)
-if((DEFINED RocksDB_DIR) AND RocksDB_DIR)
- list(APPEND RocksDB_LIBRARIES RocksDB::rocksdb)
-else()
- message("RocksDB config not found. Fallback to legacy find.")
- find_package(RocksDB)
- if(NOT ROCKSDB_FOUND)
- message(
- FATAL_ERROR
- "RocksDB not found. If you do not need caffe2_rocksdb, set "
- "-DUSE_ROCKSDB=OFF to solve this error.")
- endif()
-endif()
-
-# ---[ Third, create the CMake target.
-# The key to note is that this library will need to depend on caffe2_library,
-# which is the main lib of Caffe2. If your library explicitly depends on cuda,
-# then you will need to depend on the caffe2_gpu_library as well.
-add_library(caffe2_rocksdb ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb.cc)
-# RocksDB 7 uses C++17 STL in header.
-if(RocksDB_VERSION_MAJOR VERSION_GREATER_EQUAL 7)
- set_target_properties(caffe2_rocksdb PROPERTIES CXX_STANDARD 17)
-endif()
-target_link_libraries(caffe2_rocksdb PUBLIC torch_library)
-target_link_libraries(caffe2_rocksdb PRIVATE ${RocksDB_LIBRARIES})
-target_include_directories(caffe2_rocksdb PRIVATE ${RocksDB_INCLUDE_DIR})
-install(TARGETS caffe2_rocksdb DESTINATION lib)
-
-# ---[ Last, Append the library to Caffe2_MODULES, if we are building with
-# the main repo.
-# The purpose of this is that, for all binaries built in the Caffe2 main repo,
-# they will be built with the first class modules that are built. As a result,
-# these binaries will not need to explicitly load these modules before using
-# them.
-# Note(jiayq): this also depends on a separate cmake move to reorg test builds
-# and binary builds after modules. When it is done, this note should be removed.
-if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO)
- set(Caffe2_MODULES ${Caffe2_MODULES} caffe2_rocksdb PARENT_SCOPE)
-endif()
diff --git a/modules/rocksdb/rocksdb.cc b/modules/rocksdb/rocksdb.cc
deleted file mode 100644
index 2db4691..0000000
--- a/modules/rocksdb/rocksdb.cc
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Copyright (c) 2016-present, Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "caffe2/core/db.h"
-#include "caffe2/core/logging.h"
-#include "caffe2/core/module.h"
-#include "caffe2/core/flags.h"
-#include "rocksdb/db.h"
-#include "rocksdb/utilities/leveldb_options.h"
-
-C10_DEFINE_int(
- caffe2_rocksdb_block_size,
- 65536,
- "The caffe2 rocksdb block size when writing a rocksdb.");
-
-namespace caffe2 {
-namespace db {
-
-class RocksDBCursor : public Cursor {
- public:
- explicit RocksDBCursor(rocksdb::DB* db)
- : iter_(db->NewIterator(rocksdb::ReadOptions())) {
- SeekToFirst();
- }
- ~RocksDBCursor() {}
- void Seek(const string& key) override { iter_->Seek(key); }
- bool SupportsSeek() override { return true; }
- void SeekToFirst() override { iter_->SeekToFirst(); }
- void Next() override { iter_->Next(); }
- string key() override { return iter_->key().ToString(); }
- string value() override { return iter_->value().ToString(); }
- bool Valid() override { return iter_->Valid(); }
-
- private:
- std::unique_ptr<rocksdb::Iterator> iter_;
-};
-
-class RocksDBTransaction : public Transaction {
- public:
- explicit RocksDBTransaction(rocksdb::DB* db) : db_(db) {
- CAFFE_ENFORCE(db_);
- batch_.reset(new rocksdb::WriteBatch());
- }
- ~RocksDBTransaction() { Commit(); }
- void Put(const string& key, string&& value) override {
- batch_->Put(key, value);
- }
- void Commit() override {
- rocksdb::Status status = db_->Write(rocksdb::WriteOptions(), batch_.get());
- batch_.reset(new rocksdb::WriteBatch());
- CAFFE_ENFORCE(
- status.ok(), "Failed to write batch to rocksdb: " + status.ToString());
- }
-
- private:
- rocksdb::DB* db_;
- std::unique_ptr<rocksdb::WriteBatch> batch_;
-
- C10_DISABLE_COPY_AND_ASSIGN(RocksDBTransaction);
-};
-
-class RocksDB : public DB {
- public:
- RocksDB(const string& source, Mode mode) : DB(source, mode) {
- rocksdb::LevelDBOptions options;
- options.block_size = FLAGS_caffe2_rocksdb_block_size;
- options.write_buffer_size = 268435456;
- options.max_open_files = 100;
- options.error_if_exists = mode == NEW;
- options.create_if_missing = mode != READ;
- rocksdb::Options rocksdb_options = rocksdb::ConvertOptions(options);
-
- rocksdb::DB* db_temp;
- rocksdb::Status status = rocksdb::DB::Open(
- rocksdb_options, source, &db_temp);
- CAFFE_ENFORCE(
- status.ok(),
- "Failed to open rocksdb ",
- source,
- "\n",
- status.ToString());
- db_.reset(db_temp);
- VLOG(1) << "Opened rocksdb " << source;
- }
-
- void Close() override { db_.reset(); }
- unique_ptr<Cursor> NewCursor() override {
- return make_unique<RocksDBCursor>(db_.get());
- }
- unique_ptr<Transaction> NewTransaction() override {
- return make_unique<RocksDBTransaction>(db_.get());
- }
-
- private:
- std::unique_ptr<rocksdb::DB> db_;
-};
-
-REGISTER_CAFFE2_DB(RocksDB, RocksDB);
-// For lazy-minded, one can also call with lower-case name.
-REGISTER_CAFFE2_DB(rocksdb, RocksDB);
-
-} // namespace db
-
-CAFFE2_MODULE(caffe2_rocksdb, "RocksDB implementation for caffe2::DB.");
-} // namespace caffe2
diff --git a/scripts/build_android.sh b/scripts/build_android.sh
index b0d1dd4..0a842a2 100755
--- a/scripts/build_android.sh
+++ b/scripts/build_android.sh
@@ -128,8 +128,6 @@
CMAKE_ARGS+=("-DUSE_ITT=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
-CMAKE_ARGS+=("-DUSE_LMDB=OFF")
-CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_OPENMP=OFF")
# Only toggle if VERBOSE=1
diff --git a/scripts/build_ios.sh b/scripts/build_ios.sh
index 8262eec..abe6471 100755
--- a/scripts/build_ios.sh
+++ b/scripts/build_ios.sh
@@ -107,8 +107,6 @@
CMAKE_ARGS+=("-DUSE_ITT=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
-CMAKE_ARGS+=("-DUSE_LMDB=OFF")
-CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_NUMPY=OFF")
CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
diff --git a/scripts/build_mobile.sh b/scripts/build_mobile.sh
index 902458b..a144418 100755
--- a/scripts/build_mobile.sh
+++ b/scripts/build_mobile.sh
@@ -68,8 +68,6 @@
CMAKE_ARGS+=("-DUSE_ITT=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
-CMAKE_ARGS+=("-DUSE_LMDB=OFF")
-CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_OPENMP=OFF")
CMAKE_ARGS+=("-DUSE_MKLDNN=OFF")
diff --git a/scripts/build_tegra_x1.sh b/scripts/build_tegra_x1.sh
index b1121ff..063e17d 100755
--- a/scripts/build_tegra_x1.sh
+++ b/scripts/build_tegra_x1.sh
@@ -29,10 +29,7 @@
# obtain optional dependencies that are usually useful to have.
echo "Installing optional dependencies."
sudo apt-get install \
- libleveldb-dev \
- liblmdb-dev \
libpython-dev \
- libsnappy-dev \
python-numpy \
python-pip \
python-protobuf
diff --git a/scripts/build_tizen.sh b/scripts/build_tizen.sh
index 33fc65c..ce64b6c 100755
--- a/scripts/build_tizen.sh
+++ b/scripts/build_tizen.sh
@@ -55,9 +55,6 @@
-DRUN_HAVE_POSIX_REGEX=0 \
-DHAVE_GNU_POSIX_REGEX=0 \
-DUSE_MPI=OFF -DUSE_OPENMP=OFF \
- -DUSE_ROCKSDB=OFF \
- -DUSE_LEVELDB=OFF \
- -DUSE_LMDB=OFF \
-DBUILD_PYTHON=OFF \
-DUSE_GLOO=OFF \
-DUSE_OPENCV=OFF \
@@ -84,10 +81,7 @@
# Obtain optional dependencies that are usually useful to have.
echo "Installing optional dependencies."
sudo zypper install \
- libleveldb-dev \
- liblmdb-dev \
libpython-dev \
- libsnappy-dev \
python-numpy \
python-pip \
python-protobuf
@@ -110,7 +104,6 @@
-DUSE_CUDA=OFF \
-DUSE_ITT=OFF \
-DUSE_OPENCV=OFF \
- -DUSE_LMDB=OFF \
-DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=soft" \
|| exit 1
