Sync fbcode/caffe2 and xplat/caffe2 (1) (#19218)
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/19218
Sync some contents between fbcode/caffe2 and xplat/caffe2 to move closer towards a world where they are identical.
Reviewed By: dzhulgakov
Differential Revision: D14919916
fbshipit-source-id: 29c6b6d89ac556d58ae3cd02619aca88c79591c1
diff --git a/scripts/proto.ps1 b/scripts/proto.ps1
new file mode 100644
index 0000000..53812f9
--- /dev/null
+++ b/scripts/proto.ps1
@@ -0,0 +1,17 @@
+param(
+ [string]$protoc,
+ [string]$srcdir,
+ [string]$unprocessed,
+ [string]$processed,
+ [string]$out
+)
+$ErrorActionPreference = "Stop"
+Get-Content $unprocessed | % {$_ -Replace "caffe2/proto/caffe2.proto", "caffe2.proto"} | Set-Content $processed
+Add-Content -Path $processed -Value "option optimize_for = LITE_RUNTIME;`n" -NoNewline
+$dir = (Get-Item $processed).DirectoryName
+
+copy $srcdir/caffe2/proto/caffe2.proto $srcdir/caffe2.proto
+Add-Content -Path $srcdir/caffe2.proto -Value "option optimize_for = LITE_RUNTIME;`n" -NoNewline
+
+$cmd = "$protoc -I${dir} --cpp_out=$out $processed"
+Invoke-Expression $cmd
diff --git a/scripts/run_mobilelab.py b/scripts/run_mobilelab.py
new file mode 100755
index 0000000..ce6dcac
--- /dev/null
+++ b/scripts/run_mobilelab.py
@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+
+"""
+Script to launch all caffe2 MobileLab tests
+MobieLab uses `flow-cli` to launch jobs
+To install flow-cli on your devserver, run
+```
+sudo feature install fblearner_flow
+```
+
+Example usage:
+
+- Run all tests: ~/fbsource/xplat/caffe2/scripts/run_mobilelab.py --diff_id 56041720 --base_commit 583c0d7396c8d0469a225bf2bb19d8ecd25ba3af
+- Run all android tests: ~/fbsource/xplat/caffe2/scripts/run_mobilelab.py --diff_id 56041720 --base_commit <commit_id> --android
+- Run all android tests: ~/fbsource/xplat/caffe2/scripts/run_mobilelab.py --diff_id 56041720 --base_commit <commit_id> --iOS
+- Run all tests and let LabBot post lab run results on your diff: ~/fbsource/xplat/caffe2/scripts/run_mobilelab.py --diff_id 56041720 --base_commit 583c0d7396c8d0469a225bf2bb19d8ecd25ba3af --comment_on_revision D8756131
+
+"""
+
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+
+# comment out the tests you don't need to run
+android_tests = [
+ "android_microbenchmark.aml.microbenchmark_caffe2.correctness.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.body_tracking_cold_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.body_tracking_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.body_tracking_memory.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.body_tracking_operator_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.hand_tracking_cold_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.hand_tracking_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.hand_tracking_memory.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.hand_tracking_operator_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.person_segmentation_cold_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.person_segmentation_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.person_segmentation_memory.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.person_segmentation_operator_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_42_cold_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_42_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_42_memory.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_42_operator_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_84_v1_cold_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_84_v1_delay.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_84_v1_memory.nexus6",
+ "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_84_v1_operator_delay.nexus6",
+]
+
+iOS_tests = [
+ "xcui.caffe2.correctness.ios10.iPhone7",
+ "xcui.caffe2.body_tracking_cold_delay.ios10.iPhone7",
+ "xcui.caffe2.body_tracking_delay.ios10.iPhone7",
+ "xcui.caffe2.body_tracking_memory.ios10.iPhone7",
+ "xcui.caffe2.hand_tracking_cold_delay.ios10.iPhone7",
+ "xcui.caffe2.hand_tracking_delay.ios10.iPhone7",
+ "xcui.caffe2.hand_tracking_memory.ios10.iPhone7",
+ "xcui.caffe2.person_segmentation_cold_delay.ios10.iPhone7",
+ "xcui.caffe2.person_segmentation_delay.ios10.iPhone7",
+ "xcui.caffe2.person_segmentation_memory.ios10.iPhone7",
+ "xcui.caffe2.xray_mobile_42_cold_delay.ios10.iPhone7",
+ "xcui.caffe2.xray_mobile_42_delay.ios10.iPhone7",
+ "xcui.caffe2.xray_mobile_42_memory.ios10.iPhone7",
+ "xcui.caffe2.xray_mobile_84_v1_cold_delay.ios10.iPhone7",
+ "xcui.caffe2.xray_mobile_84_v1_delay.ios10.iPhone7",
+ "xcui.caffe2.xray_mobile_84_v1_memory.ios10.iPhone7",
+]
+
+android_job_inputs = json.loads(
+ """{
+ "test_config_name": "android_microbenchmark.aml.microbenchmark_caffe2.correctness.nexus6",
+ "workflow_inputs": {
+ "gating_overrides": null,
+ "android_microbenchmark": {
+ "experiment_store_config": {
+ "metadata": {
+ "entry_point": "frontend"
+ }
+ },
+ "one_world_priority": -10,
+ "experiment_request": {
+ "treatment": {
+ "build_request": {
+ "build_spec": {
+ "diff_id": "replaceme"
+ },
+ "priority": 0
+ }
+ },
+ "control": {
+ "build_request": {
+ "build_spec": {
+ "commit_hash": "replaceme"
+ },
+ "priority": 0
+ }
+ }
+ },
+ "comment_on_revision": "replaceme"
+ }
+ }
+}"""
+)
+
+iOS_job_inputs = json.loads(
+ """{
+ "test_config_name": "xcui.caffe2.correctness.ios10.iPhone7",
+ "workflow_inputs": {
+ "xcui": {
+ "gating_overrides": null,
+ "experiment_store_config": {
+ "metadata": {
+ "entry_point": "frontend"
+ }
+ },
+ "one_world_priority": -10,
+ "experiment_request": {
+ "treatment": {
+ "build_request": {
+ "build_spec": {
+ "diff_id": "replaceme"
+ },
+ "priority": 0
+ }
+ },
+ "control": {
+ "build_request": {
+ "build_spec": {
+ "commit_hash": "replaceme"
+ },
+ "priority": 0
+ }
+ }
+ },
+ "comment_on_revision": "replaceme"
+ }
+ }
+}"""
+)
+
+
+parser = argparse.ArgumentParser(description="Launch all caffe2 mobilelab jobs")
+parser.add_argument(
+ "--diff_id",
+ required=True,
+ help="the diff version (not the DNNNNNNNN revision) for your change, e.g. 55434539",
+)
+parser.add_argument(
+ "--base_commit",
+ required=True,
+ help="base commit hash, e.g. 583c0d7396c8d0469a225bf2bb19d8ecd25ba3af",
+)
+parser.add_argument(
+ "--comment_on_revision",
+ default=None,
+ help="the phabricator revision (the DNNNNNNNN revision), eg. D8756131 or D8756131",
+)
+parser.add_argument("--android", action="store_true", help="Run all android tests")
+parser.add_argument("--iOS", action="store_true", help="Run all iOS tests")
+
+
+def main(args):
+ options = parser.parse_args(args)
+
+ diff_id = options.diff_id
+ base_commit = options.base_commit
+ comment_on_revision = options.comment_on_revision
+ if comment_on_revision and comment_on_revision.startswith("D"):
+ comment_on_revision = comment_on_revision[1:]
+
+ # Launch all jobs if the user didn't specify android or iOS
+ run_all_jobs = True if not (options.android) and not (options.iOS) else False
+
+ # Must launch jobs from fbcode
+ os.system("cd ~/fbsource/fbcode")
+
+ if run_all_jobs or options.android:
+ for t in android_tests:
+ android_job_inputs["workflow_inputs"]["android_microbenchmark"][
+ "experiment_request"
+ ]["treatment"]["build_request"]["build_spec"]["diff_id"] = diff_id
+ android_job_inputs["workflow_inputs"]["android_microbenchmark"][
+ "experiment_request"
+ ]["control"]["build_request"]["build_spec"]["commit_hash"] = base_commit
+ android_job_inputs["workflow_inputs"]["android_microbenchmark"][
+ "comment_on_revision"
+ ] = comment_on_revision
+ android_job_inputs["test_config_name"] = t
+ subprocess.check_call(
+ [
+ "flow-cli",
+ "launch",
+ "lab.run_test.entry",
+ "--parameters-json",
+ json.dumps(android_job_inputs),
+ "--entitlement",
+ "oneworld_prn",
+ "--name",
+ "Run with test config " + t,
+ ]
+ )
+ if run_all_jobs or options.iOS:
+ for t in iOS_tests:
+ iOS_job_inputs["workflow_inputs"]["xcui"]["experiment_request"][
+ "treatment"
+ ]["build_request"]["build_spec"]["diff_id"] = diff_id
+ iOS_job_inputs["workflow_inputs"]["xcui"]["experiment_request"]["control"][
+ "build_request"
+ ]["build_spec"]["commit_hash"] = base_commit
+ iOS_job_inputs["workflow_inputs"]["xcui"][
+ "comment_on_revision"
+ ] = comment_on_revision
+ iOS_job_inputs["test_config_name"] = t
+ subprocess.check_call(
+ [
+ "flow-cli",
+ "launch",
+ "lab.run_test.entry",
+ "--parameters-json",
+ json.dumps(iOS_job_inputs),
+ "--entitlement",
+ "oneworld_prn",
+ "--name",
+ "Run with test config " + t,
+ ]
+ )
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
