Install cpp tests when built (#15000)
Summary:
This is broken out of https://github.com/pytorch/pytorch/pull/13733/
We want to install cpp tests so they can ultimately be runnable from that location for Caffe2 tests run from PyTorch builds.
cc pjh5 yf225 anderspapitto
Pull Request resolved: https://github.com/pytorch/pytorch/pull/15000
Reviewed By: pjh5
Differential Revision: D13416253
Pulled By: orionr
fbshipit-source-id: 51280be0a22557a742f90c9f303c58c35cbd4a38
diff --git a/.gitignore b/.gitignore
index 9c8bbd4..0cb2aaa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,6 +23,7 @@
aten/src/ATen/Config.h
aten/src/ATen/cuda/CUDAConfig.h
build/
+caffe2/cpp_test/
dist/
docs/src/**/*
docs/cpp/build
diff --git a/.jenkins/caffe2/build.sh b/.jenkins/caffe2/build.sh
index 643dc7e..4beb22d 100755
--- a/.jenkins/caffe2/build.sh
+++ b/.jenkins/caffe2/build.sh
@@ -229,6 +229,11 @@
exit 1
fi
+ # This is to save test binaries for testing
+ mv "$INSTALL_PREFIX/test/" "$INSTALL_PREFIX/cpp_test/"
+
+ ls $INSTALL_PREFIX
+
else
# sccache will be stuck if all cores are used for compiling
@@ -237,10 +242,12 @@
export MAX_JOBS=`expr $(nproc) - 1`
fi
- USE_LEVELDB=1 USE_LMDB=1 USE_OPENCV=1 BUILD_BINARY=1 python setup.py install --user
+ USE_LEVELDB=1 USE_LMDB=1 USE_OPENCV=1 BUILD_TEST=1 BUILD_BINARY=1 python setup.py install --user
# This is to save test binaries for testing
cp -r torch/lib/tmp_install $INSTALL_PREFIX
+ mkdir -p "$INSTALL_PREFIX/cpp_test/"
+ cp -r caffe2/test/* "$INSTALL_PREFIX/cpp_test/"
ls $INSTALL_PREFIX
diff --git a/.jenkins/caffe2/test.sh b/.jenkins/caffe2/test.sh
index c118689..b2e2df0 100755
--- a/.jenkins/caffe2/test.sh
+++ b/.jenkins/caffe2/test.sh
@@ -12,12 +12,18 @@
PYTHON="python${BASH_REMATCH[1]}"
fi
-# The prefix must mirror the setting from build.sh
-INSTALL_PREFIX="/usr/local/caffe2"
-
# Add the site-packages in the caffe2 install prefix to the PYTHONPATH
SITE_DIR=$($PYTHON -c "from distutils import sysconfig; print(sysconfig.get_python_lib(prefix=''))")
-INSTALL_SITE_DIR="${INSTALL_PREFIX}/${SITE_DIR}"
+
+# Find where Caffe2 is installed. This will be the absolute path to the
+# site-packages of the active Python installation
+INSTALL_SITE_DIR=$($PYTHON -c "from distutils import sysconfig; print(sysconfig.get_python_lib())")
+INSTALL_PREFIX="$INSTALL_SITE_DIR/caffe2"
+if [ ! -d "$INSTALL_PREFIX/cpp_test" ]; then
+ echo "Directory $INSTALL_PREFIX/cpp_test not found. Fallback to legacy location."
+ INSTALL_PREFIX="/usr/local/caffe2"
+ INSTALL_SITE_DIR="${INSTALL_PREFIX}/${SITE_DIR}"
+fi
# Skip tests in environments where they are not built/applicable
if [[ "${BUILD_ENVIRONMENT}" == *-android* ]]; then
@@ -46,20 +52,20 @@
gtest_reports_dir="${TEST_DIR}/cpp"
junit_reports_dir="${TEST_DIR}/junit_reports"
mkdir -p "$gtest_reports_dir" "$junit_reports_dir"
-for test in $(find "${INSTALL_PREFIX}/test" -executable -type f); do
+for test in $(find "${INSTALL_PREFIX}/cpp_test" -executable -type f); do
case "$test" in
# skip tests we know are hanging or bad
*/mkl_utils_test|*/aten/integer_divider_test)
continue
;;
*/scalar_tensor_test|*/basic|*/native_test)
- if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
- continue
- else
- "$test"
- fi
- ;;
- *)
+ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
+ continue
+ else
+ "$test"
+ fi
+ ;;
+ *)
# Currently, we use a mixture of gtest (caffe2) and Catch2 (ATen). While
# planning to migrate to gtest as the common PyTorch c++ test suite, we
# currently do NOT use the xml test reporter, because Catch doesn't
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 159b153..7021b86 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -75,7 +75,7 @@
"NOT BUILD_SHARED_LIBS" OFF)
option(BUILD_TEST "Build C++ test binaries (need gtest and gbenchmark)" OFF)
cmake_dependent_option(
- INSTALL_TEST "Install test binaries if BUILD_TEST is on" OFF
+ INSTALL_TEST "Install test binaries if BUILD_TEST is on" ON
"BUILD_TEST" OFF)
option(USE_ACL "Use ARM Compute Library" OFF)
option(USE_ASAN "Use Address Sanitizer" OFF)
diff --git a/setup.py b/setup.py
index aa501af..c7028d1 100644
--- a/setup.py
+++ b/setup.py
@@ -1025,7 +1025,7 @@
'share/cmake/Torch/*.cmake',
],
'caffe2': [
- rel_site_packages + '/caffe2/**/*.py'
+ 'cpp_test/*',
]
},
)
diff --git a/tools/build_pytorch_libs.sh b/tools/build_pytorch_libs.sh
index 8ab575c..04d7695 100755
--- a/tools/build_pytorch_libs.sh
+++ b/tools/build_pytorch_libs.sh
@@ -342,4 +342,23 @@
if [ -d "$INSTALL_DIR/bin/" ]; then
$SYNC_COMMAND -r "$INSTALL_DIR/bin/"/* .
fi
+
+# Copy the test files to pytorch/caffe2 manually
+# They were built in pytorch/torch/lib/tmp_install/test
+# Why do we do this? So, setup.py has this section called 'package_data' which
+# you need to specify to include non-default files (usually .py files).
+# package_data takes a map from 'python package' to 'globs of files to
+# include'. By 'python package', it means a folder with an __init__.py file
+# that's not excluded in the find_packages call earlier in setup.py. So to
+# include our cpp_test into the site-packages folder in
+# site-packages/caffe2/cpp_test, we have to copy the cpp_test folder into the
+# root caffe2 folder and then tell setup.py to include them. Having another
+# folder like site-packages/caffe2_cpp_test would also be possible by adding a
+# caffe2_cpp_test folder to pytorch with an __init__.py in it.
+if [[ "$INSTALL_TEST" == "ON" ]]; then
+ echo "Copying $INSTALL_DIR/test to $BASE_DIR/caffe2/cpp_test"
+ mkdir -p "$BASE_DIR/caffe2/cpp_test/"
+ $SYNC_COMMAND -r "$INSTALL_DIR/test/"/* "$BASE_DIR/caffe2/cpp_test/"
+fi
+
popd > /dev/null
