Keep same autogenerated files structure between fbcode and OSS builds (#35951)
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/35951
Change generate_code to keep folder structure the same regardless of whether install path is provide
Amend build_variables.bzl accordingly
Another preliminary step to merge https://github.com/pytorch/pytorch/pull/35220
Test Plan: CI
Reviewed By: EscapeZero, seemethere
Differential Revision: D20839410
fbshipit-source-id: 02297560a7e48aa7c6271f7a8517fc4a1ab35271
diff --git a/tools/autograd/templates/python_functions.cpp b/tools/autograd/templates/python_functions.cpp
index 34f1321..858029c 100644
--- a/tools/autograd/templates/python_functions.cpp
+++ b/tools/autograd/templates/python_functions.cpp
@@ -5,7 +5,7 @@
#include <Python.h>
#include <ATen/ATen.h>
-#include "Functions.h"
+#include "torch/csrc/autograd/generated/Functions.h"
#include "torch/csrc/autograd/python_cpp_function.h"
namespace torch { namespace autograd { namespace generated {
diff --git a/tools/build_variables.bzl b/tools/build_variables.bzl
index 51c10cd..7dbe7bf 100644
--- a/tools/build_variables.bzl
+++ b/tools/build_variables.bzl
@@ -1,37 +1,36 @@
-# In the open-source build, these are generated into
-# torch/csrc/{autgrad,jit}/generated. In fbcode, this distinction is
-# not currently relevant so they are combined into one list.
+# In both open-source and fbcode builds, these are generated into
+# torch/csrc/{autgrad,jit}/generated.i
load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
load("@fbcode_macros//build_defs:cpp_python_extension.bzl", "cpp_python_extension")
load("@fbsource//xplat/caffe2/caffe2/fb:defs_gpu.bzl", "gpu_library_selector")
GENERATED_CPP = [
- "Functions.cpp",
- "VariableType_0.cpp",
- "VariableType_1.cpp",
- "VariableType_2.cpp",
- "VariableType_3.cpp",
- "VariableType_4.cpp",
- "register_aten_ops_0.cpp",
- "register_aten_ops_1.cpp",
- "register_aten_ops_2.cpp",
- "python_functions.cpp",
- "python_nn_functions.cpp",
- "python_torch_functions.cpp",
- "python_variable_methods.cpp",
+ "autograd/generated/Functions.cpp",
+ "autograd/generated/VariableType_0.cpp",
+ "autograd/generated/VariableType_1.cpp",
+ "autograd/generated/VariableType_2.cpp",
+ "autograd/generated/VariableType_3.cpp",
+ "autograd/generated/VariableType_4.cpp",
+ "jit/generated/register_aten_ops_0.cpp",
+ "jit/generated/register_aten_ops_1.cpp",
+ "jit/generated/register_aten_ops_2.cpp",
+ "autograd/generated/python_functions.cpp",
+ "autograd/generated/python_nn_functions.cpp",
+ "autograd/generated/python_torch_functions.cpp",
+ "autograd/generated/python_variable_methods.cpp",
]
# copied from https://github.com/pytorch/pytorch/blob/master/tools/cpp_build/torch/CMakeLists.txt
libtorch_sources = [
- ":generate-code=Functions.cpp",
- ":generate-code=register_aten_ops_0.cpp",
- ":generate-code=register_aten_ops_1.cpp",
- ":generate-code=register_aten_ops_2.cpp",
- ":generate-code=VariableType_0.cpp",
- ":generate-code=VariableType_1.cpp",
- ":generate-code=VariableType_2.cpp",
- ":generate-code=VariableType_3.cpp",
- ":generate-code=VariableType_4.cpp",
+ ":generate-code=autograd/generated/Functions.cpp",
+ ":generate-code=jit/generated/register_aten_ops_0.cpp",
+ ":generate-code=jit/generated/register_aten_ops_1.cpp",
+ ":generate-code=jit/generated/register_aten_ops_2.cpp",
+ ":generate-code=autograd/generated/VariableType_0.cpp",
+ ":generate-code=autograd/generated/VariableType_1.cpp",
+ ":generate-code=autograd/generated/VariableType_2.cpp",
+ ":generate-code=autograd/generated/VariableType_3.cpp",
+ ":generate-code=autograd/generated/VariableType_4.cpp",
"torch/csrc/autograd/VariableTypeManual.cpp",
"torch/csrc/autograd/anomaly_mode.cpp",
"torch/csrc/autograd/autograd.cpp",
@@ -331,10 +330,10 @@
torch_cpp_headers = native.glob(["torch/csrc/api/include/**/*.h"]) + ["torch/script.h"]
libtorch_python_sources = [
- ":generate-code=python_functions.cpp",
- ":generate-code=python_nn_functions.cpp",
- ":generate-code=python_torch_functions.cpp",
- ":generate-code=python_variable_methods.cpp",
+ ":generate-code=autograd/generated/python_functions.cpp",
+ ":generate-code=autograd/generated/python_nn_functions.cpp",
+ ":generate-code=autograd/generated/python_torch_functions.cpp",
+ ":generate-code=autograd/generated/python_variable_methods.cpp",
"torch/csrc/CudaIPCTypes.cpp",
"torch/csrc/DataLoader.cpp",
"torch/csrc/Device.cpp",
diff --git a/tools/setup_helpers/generate_code.py b/tools/setup_helpers/generate_code.py
index 5ab6f2b..6a73264 100644
--- a/tools/setup_helpers/generate_code.py
+++ b/tools/setup_helpers/generate_code.py
@@ -34,27 +34,32 @@
from tools.jit.gen_jit_dispatch import gen_jit_dispatch
# Build ATen based Variable classes
- autograd_gen_dir = install_dir or 'torch/csrc/autograd/generated'
- jit_gen_dir = install_dir or 'torch/csrc/jit/generated'
+ install_dir = install_dir or 'torch/csrc'
+ autograd_gen_dir = os.path.join(install_dir, 'autograd', 'generated')
+ jit_gen_dir = os.path.join(install_dir, 'jit', 'generated')
for d in (autograd_gen_dir, jit_gen_dir):
if not os.path.exists(d):
os.makedirs(d)
+ runfiles_dir = os.environ.get("RUNFILES_DIR", None)
+ data_dir = os.path.join(runfiles_dir, 'pytorch') if runfiles_dir else ''
+ autograd_dir = os.path.join(data_dir, 'tools', 'autograd')
+ tools_jit_templates = os.path.join(data_dir, 'tools', 'jit', 'templates')
if subset == "pybindings" or not subset:
- gen_autograd_python(declarations_path or DECLARATIONS_PATH, autograd_gen_dir, 'tools/autograd')
+ gen_autograd_python(declarations_path or DECLARATIONS_PATH, autograd_gen_dir, autograd_dir)
if subset == "libtorch" or not subset:
gen_autograd(
declarations_path or DECLARATIONS_PATH,
autograd_gen_dir,
- 'tools/autograd',
+ autograd_dir,
disable_autograd=disable_autograd,
disable_trace=disable_trace,
)
gen_jit_dispatch(
declarations_path or DECLARATIONS_PATH,
jit_gen_dir,
- 'tools/jit/templates',
+ tools_jit_templates,
disable_autograd=disable_autograd,
selected_op_list_path=selected_op_list_path)
