Re-add benchmarking files to ios TestApp (#85539)
Fixes #76033
The benchmarking code in the iOS TestApp was removed a while back as dead code:
https://github.com/pytorch/pytorch/pull/64849
I believe this was done in error - as this leaves our TestApp empty, nothing occurs when it runs. And we still have a tutorial up demonstrating how to use the benchmarking feature of the TestApp.
This diff restores the files that were deleted, with some minor tweaks for compatibility with changes that have happened since they were deleted.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/85539
Approved by: https://github.com/kimishpatel
diff --git a/ios/TestApp/README.md b/ios/TestApp/README.md
index 80f9531..a249957 100644
--- a/ios/TestApp/README.md
+++ b/ios/TestApp/README.md
@@ -49,3 +49,15 @@
There's no debug information in simulator test (project TestAppTests). You can copy the failed test code to
TestApp/TestApp/ViewController.mm and debug in the main TestApp.
+
+### Benchmark
+
+The benchmark folder contains two scripts that help you setup the benchmark project. The `setup.rb` does the heavy-lifting jobs of setting up the XCode project, whereas the `trace_model.py` is a Python script that you can tweak to generate your model for benchmarking. Simply follow the steps below to setup the project
+
+1. In the PyTorch root directory, run `IOS_ARCH=arm64 ./scripts/build_ios.sh` to generate the custom build from **Master** branch
+2. Navigate to the `benchmark` folder, run `python trace_model.py` to generate your model.
+3. In the same directory, open `config.json`. Those are the input parameters you can tweak.
+4. Again, in the same directory, run `ruby setup.rb` to setup the XCode project.
+5. Open the `TestApp.xcodeproj`, you're ready to go.
+
+The benchmark code is written in C++, you can use `UI_LOG` to visualize the log. See `benchmark.mm` for more details.
diff --git a/ios/TestApp/TestApp.xcodeproj/project.pbxproj b/ios/TestApp/TestApp.xcodeproj/project.pbxproj
index c516d91..09aeead 100644
--- a/ios/TestApp/TestApp.xcodeproj/project.pbxproj
+++ b/ios/TestApp/TestApp.xcodeproj/project.pbxproj
@@ -7,6 +7,7 @@
objects = {
/* Begin PBXBuildFile section */
+ 4C636CFD28DDAE0200FF9B4D /* Benchmark.mm in Sources */ = {isa = PBXBuildFile; fileRef = 4C636CFB28DDAE0200FF9B4D /* Benchmark.mm */; };
A06D4CB5232F0DB200763E16 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = A06D4CB4232F0DB200763E16 /* AppDelegate.m */; };
A06D4CB8232F0DB200763E16 /* ViewController.mm in Sources */ = {isa = PBXBuildFile; fileRef = A06D4CB7232F0DB200763E16 /* ViewController.mm */; };
A06D4CBB232F0DB200763E16 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = A06D4CB9232F0DB200763E16 /* Main.storyboard */; };
@@ -26,6 +27,8 @@
/* End PBXContainerItemProxy section */
/* Begin PBXFileReference section */
+ 4C636CFB28DDAE0200FF9B4D /* Benchmark.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = Benchmark.mm; sourceTree = "<group>"; };
+ 4C636CFC28DDAE0200FF9B4D /* Benchmark.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Benchmark.h; sourceTree = "<group>"; };
A06D4CB0232F0DB200763E16 /* TestApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TestApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
A06D4CB3232F0DB200763E16 /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = "<group>"; };
A06D4CB4232F0DB200763E16 /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = "<group>"; };
@@ -79,6 +82,8 @@
A06D4CB2232F0DB200763E16 /* TestApp */ = {
isa = PBXGroup;
children = (
+ 4C636CFC28DDAE0200FF9B4D /* Benchmark.h */,
+ 4C636CFB28DDAE0200FF9B4D /* Benchmark.mm */,
A06D4CB3232F0DB200763E16 /* AppDelegate.h */,
A06D4CB4232F0DB200763E16 /* AppDelegate.m */,
A06D4CB6232F0DB200763E16 /* ViewController.h */,
@@ -146,12 +151,12 @@
attributes = {
LastUpgradeCheck = 1030;
TargetAttributes = {
- A06D4CAF232F0DB200763E16 = {
- CreatedOnToolsVersion = 10.3;
- };
- A0EA3AFE237FCB08007CEA34 = {
- CreatedOnToolsVersion = 11.2.1;
- };
+ A06D4CAF232F0DB200763E16 = {
+ CreatedOnToolsVersion = 10.3;
+ };
+ A0EA3AFE237FCB08007CEA34 = {
+ CreatedOnToolsVersion = 11.2.1;
+ };
};
};
buildConfigurationList = A06D4CAB232F0DB200763E16 /* Build configuration list for PBXProject "TestApp" */;
@@ -201,6 +206,7 @@
A06D4CB8232F0DB200763E16 /* ViewController.mm in Sources */,
A06D4CC3232F0DB200763E16 /* main.m in Sources */,
A06D4CB5232F0DB200763E16 /* AppDelegate.m in Sources */,
+ 4C636CFD28DDAE0200FF9B4D /* Benchmark.mm in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -283,8 +289,8 @@
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
- "DEBUG=1",
- "$(inherited)",
+ "DEBUG=1",
+ "$(inherited)",
);
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
@@ -362,8 +368,8 @@
ENABLE_BITCODE = NO;
INFOPLIST_FILE = TestApp/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
- "$(inherited)",
- "@executable_path/Frameworks",
+ "$(inherited)",
+ "@executable_path/Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = com.pytorch.ios.TestApp;
PRODUCT_NAME = "$(TARGET_NAME)";
@@ -381,8 +387,8 @@
ENABLE_BITCODE = NO;
INFOPLIST_FILE = TestApp/Info.plist;
LD_RUNPATH_SEARCH_PATHS = (
- "$(inherited)",
- "@executable_path/Frameworks",
+ "$(inherited)",
+ "@executable_path/Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = com.pytorch.ios.TestApp;
PRODUCT_NAME = "$(TARGET_NAME)";
@@ -400,9 +406,9 @@
INFOPLIST_FILE = TestAppTests/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 12.4;
LD_RUNPATH_SEARCH_PATHS = (
- "$(inherited)",
- "@executable_path/Frameworks",
- "@loader_path/Frameworks",
+ "$(inherited)",
+ "@executable_path/Frameworks",
+ "@loader_path/Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = com.pytorch.ios.TestAppTests;
PRODUCT_NAME = "$(TARGET_NAME)";
@@ -419,9 +425,9 @@
INFOPLIST_FILE = TestAppTests/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 12.4;
LD_RUNPATH_SEARCH_PATHS = (
- "$(inherited)",
- "@executable_path/Frameworks",
- "@loader_path/Frameworks",
+ "$(inherited)",
+ "@executable_path/Frameworks",
+ "@loader_path/Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = com.pytorch.ios.TestAppTests;
PRODUCT_NAME = "$(TARGET_NAME)";
diff --git a/ios/TestApp/TestApp/Benchmark.h b/ios/TestApp/TestApp/Benchmark.h
new file mode 100644
index 0000000..cd45665
--- /dev/null
+++ b/ios/TestApp/TestApp/Benchmark.h
@@ -0,0 +1,12 @@
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface Benchmark : NSObject
+
++ (BOOL)setup:(NSDictionary* )config;
++ (NSString* )run;
+
+@end
+
+NS_ASSUME_NONNULL_END
diff --git a/ios/TestApp/TestApp/Benchmark.mm b/ios/TestApp/TestApp/Benchmark.mm
new file mode 100644
index 0000000..2070eff
--- /dev/null
+++ b/ios/TestApp/TestApp/Benchmark.mm
@@ -0,0 +1,105 @@
+#import "Benchmark.h"
+#include <string>
+#include <vector>
+#include <torch/script.h>
+#include <torch/csrc/jit/mobile/function.h>
+#include <torch/csrc/jit/mobile/import.h>
+#include <torch/csrc/jit/mobile/interpreter.h>
+#include <torch/csrc/jit/mobile/module.h>
+#include <torch/csrc/jit/mobile/observer.h>
+#include "ATen/ATen.h"
+#include "caffe2/core/timer.h"
+#include "caffe2/utils/string_utils.h"
+#include "torch/csrc/autograd/grad_mode.h"
+
+static std::string model = "model_lite.ptl";
+static std::string input_dims = "1,3,224,224";
+static std::string input_type = "float";
+static BOOL print_output = false;
+static int warmup = 10;
+static int iter = 10;
+
+@implementation Benchmark
+
++ (BOOL)setup:(NSDictionary*)config {
+ NSString* modelPath = [[NSBundle mainBundle] pathForResource:@"model_lite" ofType:@"ptl"];
+ if (![[NSFileManager defaultManager] fileExistsAtPath:modelPath]) {
+ NSLog(@"model_lite.ptl doesn't exist!");
+ return NO;
+ }
+ model = std::string(modelPath.UTF8String);
+ input_dims = std::string(((NSString*)config[@"input_dims"]).UTF8String);
+ input_type = std::string(((NSString*)config[@"input_type"]).UTF8String);
+ warmup = ((NSNumber*)config[@"warmup"]).intValue;
+ iter = ((NSNumber*)config[@"iter"]).intValue;
+ print_output = ((NSNumber*)config[@"print_output"]).boolValue;
+ return YES;
+}
+
++ (NSString*)run {
+ std::vector<std::string> logs;
+#define UI_LOG(fmt, ...) \
+ { \
+ NSString* log = [NSString stringWithFormat:fmt, __VA_ARGS__]; \
+ NSLog(@"%@", log); \
+ logs.push_back(log.UTF8String); \
+ }
+
+ CAFFE_ENFORCE_GE(input_dims.size(), 0, "Input dims must be specified.");
+ CAFFE_ENFORCE_GE(input_type.size(), 0, "Input type must be specified.");
+
+ std::vector<std::string> input_dims_list = caffe2::split(';', input_dims);
+ std::vector<std::string> input_type_list = caffe2::split(';', input_type);
+ CAFFE_ENFORCE_EQ(input_dims_list.size(), input_type_list.size(),
+ "Input dims and type should have the same number of items.");
+
+ std::vector<c10::IValue> inputs;
+ for (size_t i = 0; i < input_dims_list.size(); ++i) {
+ auto input_dims_str = caffe2::split(',', input_dims_list[i]);
+ std::vector<int64_t> input_dims;
+ for (const auto& s : input_dims_str) {
+ input_dims.push_back(c10::stoi(s));
+ }
+ if (input_type_list[i] == "float") {
+ inputs.push_back(torch::ones(input_dims, at::ScalarType::Float));
+ } else if (input_type_list[i] == "uint8_t") {
+ inputs.push_back(torch::ones(input_dims, at::ScalarType::Byte));
+ } else {
+ CAFFE_THROW("Unsupported input type: ", input_type_list[i]);
+ }
+ }
+
+ c10::InferenceMode mode;
+ auto module = torch::jit::_load_for_mobile(model);
+
+// module.eval();
+ if (print_output) {
+ std::cout << module.forward(inputs) << std::endl;
+ }
+ UI_LOG(@"Running warmup runs", nil);
+ CAFFE_ENFORCE(warmup >= 0, "Number of warm up runs should be non negative, provided ", warmup,
+ ".");
+ for (int i = 0; i < warmup; ++i) {
+ module.forward(inputs);
+ }
+ UI_LOG(@"Main runs", nil);
+ CAFFE_ENFORCE(iter >= 0, "Number of main runs should be non negative, provided ", iter, ".");
+ caffe2::Timer timer;
+ auto millis = timer.MilliSeconds();
+ for (int i = 0; i < iter; ++i) {
+ module.forward(inputs);
+ }
+ millis = timer.MilliSeconds();
+ UI_LOG(@"Main run finished. Milliseconds per iter: %.3f", millis / iter, nil);
+ UI_LOG(@"Iters per second: : %.3f", 1000.0 * iter / millis, nil);
+ UI_LOG(@"Done.", nil);
+
+ NSString* results = @"";
+ for (auto& msg : logs) {
+ results = [results stringByAppendingString:[NSString stringWithUTF8String:msg.c_str()]];
+ results = [results stringByAppendingString:@"\n"];
+ }
+ return results;
+}
+
+@end
diff --git a/ios/TestApp/TestApp/ViewController.mm b/ios/TestApp/TestApp/ViewController.mm
index d8ecacd..89ba120 100644
--- a/ios/TestApp/TestApp/ViewController.mm
+++ b/ios/TestApp/TestApp/ViewController.mm
@@ -1,12 +1,45 @@
#import "ViewController.h"
+#import "Benchmark.h"
+
@interface ViewController ()
+@property(nonatomic, strong) UITextView* textView;
@end
@implementation ViewController
- (void)viewDidLoad {
[super viewDidLoad];
+
+ self.textView = [[UITextView alloc] initWithFrame:self.view.bounds];
+ self.textView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
+ [self.view addSubview:self.textView];
+
+ NSData* configData = [NSData dataWithContentsOfFile:[[NSBundle mainBundle] pathForResource:@"config" ofType:@"json"]];
+ if (!configData) {
+ NSLog(@"Config.json not found!");
+ return;
+ }
+
+ NSError* err;
+ NSDictionary* config = [NSJSONSerialization JSONObjectWithData:configData options:NSJSONReadingAllowFragments error:&err];
+ if (err) {
+ NSLog(@"Parse config.json failed!");
+ return;
+ }
+
+ [Benchmark setup:config];
+ [self runBenchmark];
+}
+
+- (void)runBenchmark {
+ self.textView.text = @"Start benchmarking...\n";
+ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+ NSString* text = [Benchmark run];
+ dispatch_async(dispatch_get_main_queue(), ^{
+ self.textView.text = [self.textView.text stringByAppendingString:text];
+ });
+ });
}
@end
diff --git a/ios/TestApp/benchmark/config.json b/ios/TestApp/benchmark/config.json
new file mode 100644
index 0000000..f7b991c
--- /dev/null
+++ b/ios/TestApp/benchmark/config.json
@@ -0,0 +1,7 @@
+{
+ "input_dims": "1,3,224,224",
+ "input_type": "float",
+ "warmup": 10,
+ "iter": 10,
+ "print_output": false
+}
diff --git a/ios/TestApp/benchmark/setup.rb b/ios/TestApp/benchmark/setup.rb
index 82c155f..b7e64bd 100644
--- a/ios/TestApp/benchmark/setup.rb
+++ b/ios/TestApp/benchmark/setup.rb
@@ -46,7 +46,8 @@
group.set_source_tree('SOURCE_ROOT')
group.files.each do |file|
if (file.name.to_s.end_with?(".pt") ||
- file.name.to_s.end_with?(".ptl"))
+ file.name.to_s.end_with?(".ptl") ||
+ file.name == "config.json")
group.remove_reference(file)
targets.each do |target|
target.resources_build_phase.remove_file_reference(file)
@@ -54,6 +55,12 @@
end
end
+config_path = File.expand_path("./config.json")
+if not File.exist?(config_path)
+ raise "config.json can't be found!"
+end
+config_file_ref = group.new_reference(config_path)
+
file_refs = []
# collect models
puts "Installing models..."
@@ -66,6 +73,7 @@
end
targets.each do |target|
+ target.resources_build_phase.add_file_reference(config_file_ref, true)
file_refs.each do |ref|
target.resources_build_phase.add_file_reference(ref, true)
end
