Remove distutils (#57040)
Summary:
[distutils](https://docs.python.org/3/library/distutils.html) is on its way out and will be deprecated-on-import for Python 3.10+ and removed in Python 3.12 (see [PEP 632](https://www.python.org/dev/peps/pep-0632/)). There's no reason for us to keep it around since all the functionality we want from it can be found in `setuptools` / `sysconfig`. `setuptools` includes a copy of most of `distutils` (which is fine to use according to the PEP), that it uses under the hood, so this PR also uses that in some places.
Fixes #56527
Pull Request resolved: https://github.com/pytorch/pytorch/pull/57040
Pulled By: driazati
Reviewed By: nikithamalgifb
Differential Revision: D28051356
fbshipit-source-id: 1ca312219032540e755593e50da0c9e23c62d720
diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
index 9c562bc..212d6b26 100644
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -1706,8 +1706,9 @@
# https://github.com/pytorch/pytorch/tree/master/tools/build_pytorch_libs.bat#note-backslash-munging-on-windows
pycmd(PYTHON_SITE_PACKAGES "
import os
- from distutils import sysconfig
- print(sysconfig.get_python_lib(prefix=''))
+ import sysconfig
+ relative_site_packages = sysconfig.get_path('purelib').replace(sysconfig.get_path('data'), '').lstrip(os.path.sep)
+ print(relative_site_packages)
")
file(TO_CMAKE_PATH ${PYTHON_SITE_PACKAGES} PYTHON_SITE_PACKAGES)
set(PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES} PARENT_SCOPE) # for Summary
@@ -1718,9 +1719,21 @@
# Try to get from python through sysconfig.get_env_var('EXT_SUFFIX') first,
# fallback to ".pyd" if windows and ".so" for all others.
pycmd(PY_EXT_SUFFIX "
- from distutils import sysconfig
- ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
- print(ext_suffix if ext_suffix else '')
+ def get_ext_suffix():
+ import sys
+ if sys.version_info < (3, 8) and sys.platform == 'win32':
+ # Workaround for https://bugs.python.org/issue39825
+ import _imp
+ return _imp.extension_suffixes()[0]
+ else:
+ import sysconfig
+ return sysconfig.get_config_var('EXT_SUFFIX')
+
+ suffix = get_ext_suffix()
+ if suffix is not None:
+ print(suffix)
+ else:
+ print()
")
if("${PY_EXT_SUFFIX}" STREQUAL "")
if(MSVC)
diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
index dc93772c..59a7be9 100644
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@@ -935,23 +935,18 @@
# then these will just use "python", but at least they'll be consistent with
# each other).
if(NOT DEFINED PYTHON_INCLUDE_DIR)
- # distutils.sysconfig, if it's installed, is more accurate than sysconfig,
- # which sometimes outputs directories that do not exist
- pycmd_no_exit(_py_inc _exitcode "from distutils import sysconfig; print(sysconfig.get_python_inc())")
+ # TODO: Verify that sysconfig isn't inaccurate
+ pycmd_no_exit(_py_inc _exitcode "import sysconfig; print(sysconfig.get_path('include'))")
if("${_exitcode}" EQUAL 0 AND IS_DIRECTORY "${_py_inc}")
set(PYTHON_INCLUDE_DIR "${_py_inc}")
- message(STATUS "Setting Python's include dir to ${_py_inc} from distutils.sysconfig")
+ message(STATUS "Setting Python's include dir to ${_py_inc} from sysconfig")
else()
- pycmd_no_exit(_py_inc _exitcode "from sysconfig import get_paths; print(get_paths()['include'])")
- if("${_exitcode}" EQUAL 0 AND IS_DIRECTORY "${_py_inc}")
- set(PYTHON_INCLUDE_DIR "${_py_inc}")
- message(STATUS "Setting Python's include dir to ${_py_inc} from sysconfig")
- endif()
+ message(WARNING "Could not set Python's include dir to ${_py_inc} from sysconfig")
endif()
endif(NOT DEFINED PYTHON_INCLUDE_DIR)
if(NOT DEFINED PYTHON_LIBRARY)
- pycmd_no_exit(_py_lib _exitcode "from sysconfig import get_paths; print(get_paths()['stdlib'])")
+ pycmd_no_exit(_py_lib _exitcode "import sysconfig; print(sysconfig.get_path('stdlib'))")
if("${_exitcode}" EQUAL 0 AND EXISTS "${_py_lib}" AND EXISTS "${_py_lib}")
set(PYTHON_LIBRARY "${_py_lib}")
if(MSVC)
diff --git a/scripts/build_android.sh b/scripts/build_android.sh
index 5a8cf73..90f8f92 100755
--- a/scripts/build_android.sh
+++ b/scripts/build_android.sh
@@ -61,7 +61,7 @@
if [ -z "${BUILD_CAFFE2_MOBILE:-}" ]; then
# Build PyTorch mobile
- CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
+ CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
# custom build with selected ops
diff --git a/scripts/build_ios.sh b/scripts/build_ios.sh
index 21d05f6..54e2812 100755
--- a/scripts/build_ios.sh
+++ b/scripts/build_ios.sh
@@ -13,7 +13,7 @@
if [ -z "${BUILD_CAFFE2_MOBILE:-}" ]; then
# Build PyTorch mobile
- CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
+ CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
# custom build with selected ops
diff --git a/scripts/build_mobile.sh b/scripts/build_mobile.sh
index c413f86..bc87474 100755
--- a/scripts/build_mobile.sh
+++ b/scripts/build_mobile.sh
@@ -15,7 +15,7 @@
echo "Caffe2 path: $CAFFE2_ROOT"
CMAKE_ARGS=()
-CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
+CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF")
diff --git a/scripts/get_python_cmake_flags.py b/scripts/get_python_cmake_flags.py
index 9121c5e..131f5b7 100644
--- a/scripts/get_python_cmake_flags.py
+++ b/scripts/get_python_cmake_flags.py
@@ -15,12 +15,12 @@
-from distutils import sysconfig
+import sysconfig
import sys
flags = [
'-DPYTHON_EXECUTABLE:FILEPATH={}'.format(sys.executable),
- '-DPYTHON_INCLUDE_DIR={}'.format(sysconfig.get_python_inc()),
+ '-DPYTHON_INCLUDE_DIR={}'.format(sysconfig.get_path('include')),
]
print(' '.join(flags), end='')
diff --git a/setup.py b/setup.py
index a3f416f..1bfb7a7 100644
--- a/setup.py
+++ b/setup.py
@@ -21,9 +21,7 @@
# default behavior of autogoo and cmake build systems.)
#
# CC
-# the C/C++ compiler to use (NB: the CXX flag has no effect for distutils
-# compiles, because distutils always uses CC to compile, even for C++
-# files.
+# the C/C++ compiler to use
#
# Environment variables for feature toggles:
#
@@ -197,14 +195,10 @@
from setuptools import setup, Extension, find_packages
from collections import defaultdict
-from distutils import core
-from distutils.core import Distribution
-from distutils.errors import DistutilsArgError
+from setuptools.dist import Distribution
import setuptools.command.build_ext
import setuptools.command.install
-import distutils.command.clean
-import distutils.command.sdist
-import distutils.sysconfig
+import setuptools.command.sdist
import filecmp
import shutil
import subprocess
@@ -213,6 +207,7 @@
import glob
import importlib
import time
+import sysconfig
from tools.build_pytorch_libs import build_caffe2
from tools.setup_helpers.env import (IS_WINDOWS, IS_DARWIN, IS_LINUX,
@@ -261,31 +256,31 @@
def report(*args):
pass
+ # Make distutils respect --quiet too
+ setuptools.distutils.log.warn = report
+
# Constant known variables used throughout this file
cwd = os.path.dirname(os.path.abspath(__file__))
lib_path = os.path.join(cwd, "torch", "lib")
third_party_path = os.path.join(cwd, "third_party")
caffe2_build_dir = os.path.join(cwd, "build")
-# lib/pythonx.x/site-packages
-rel_site_packages = distutils.sysconfig.get_python_lib(prefix='')
-# full absolute path to the dir above
-full_site_packages = distutils.sysconfig.get_python_lib()
+
# CMAKE: full path to python library
if IS_WINDOWS:
cmake_python_library = "{}/libs/python{}.lib".format(
- distutils.sysconfig.get_config_var("prefix"),
- distutils.sysconfig.get_config_var("VERSION"))
+ sysconfig.get_config_var("prefix"),
+ sysconfig.get_config_var("VERSION"))
# Fix virtualenv builds
# TODO: Fix for python < 3.3
if not os.path.exists(cmake_python_library):
cmake_python_library = "{}/libs/python{}.lib".format(
sys.base_prefix,
- distutils.sysconfig.get_config_var("VERSION"))
+ sysconfig.get_config_var("VERSION"))
else:
cmake_python_library = "{}/{}".format(
- distutils.sysconfig.get_config_var("LIBDIR"),
- distutils.sysconfig.get_config_var("INSTSONAME"))
-cmake_python_include_dir = distutils.sysconfig.get_python_inc()
+ sysconfig.get_config_var("LIBDIR"),
+ sysconfig.get_config_var("INSTSONAME"))
+cmake_python_include_dir = sysconfig.get_path("include")
################################################################################
@@ -491,9 +486,9 @@
# Do not use clang to compile extensions if `-fstack-clash-protection` is defined
# in system CFLAGS
- system_c_flags = str(distutils.sysconfig.get_config_var('CFLAGS'))
- if IS_LINUX and '-fstack-clash-protection' in system_c_flags and 'clang' in os.environ.get('CC', ''):
- os.environ['CC'] = str(distutils.sysconfig.get_config_var('CC'))
+ c_flags = str(os.getenv('CFLAGS', ''))
+ if IS_LINUX and '-fstack-clash-protection' in c_flags and 'clang' in os.environ.get('CC', ''):
+ os.environ['CC'] = str(os.environ['CC'])
# It's an old-style class in Python 2.7...
setuptools.command.build_ext.build_ext.run(self)
@@ -547,7 +542,8 @@
filename = self.get_ext_filename(fullname)
report("\nCopying extension {}".format(ext.name))
- src = os.path.join("torch", rel_site_packages, filename)
+ relative_site_packages = sysconfig.get_path('purelib').replace(sysconfig.get_path('data'), '').lstrip(os.path.sep)
+ src = os.path.join("torch", relative_site_packages, filename)
if not os.path.exists(src):
report("{} does not exist".format(src))
del self.extensions[i]
@@ -559,10 +555,11 @@
os.makedirs(dst_dir)
self.copy_file(src, dst)
i += 1
- distutils.command.build_ext.build_ext.build_extensions(self)
+ setuptools.command.build_ext.build_ext.build_extensions(self)
+
def get_outputs(self):
- outputs = distutils.command.build_ext.build_ext.get_outputs(self)
+ outputs = setuptools.command.build_ext.build_ext.get_outputs(self)
outputs.append(os.path.join(self.build_lib, "caffe2"))
report("setup.py::get_outputs returning {}".format(outputs))
return outputs
@@ -644,7 +641,15 @@
super().run()
-class clean(distutils.command.clean.clean):
+class clean(setuptools.Command):
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
def run(self):
import glob
import re
@@ -665,10 +670,8 @@
except OSError:
shutil.rmtree(filename, ignore_errors=True)
- super().run()
-
-class sdist(distutils.command.sdist.sdist):
+class sdist(setuptools.command.sdist.sdist):
def run(self):
with concat_license_files():
super().run()
@@ -863,14 +866,11 @@
# Parse the command line and check the arguments
# before we proceed with building deps and setup
dist = Distribution()
- dist.script_name = sys.argv[0]
- dist.script_args = sys.argv[1:]
try:
- ok = dist.parse_command_line()
- except DistutilsArgError as msg:
- raise SystemExit(core.gen_usage(dist.script_name) + "\nerror: %s" % msg)
- if not ok:
- sys.exit()
+ dist.parse_command_line()
+ except setuptools.distutils.errors.DistutilsArgError as e:
+ print(e)
+ sys.exit(1)
if RUN_BUILD_DEPS:
build_deps()
diff --git a/test/test_spectral_ops.py b/test/test_spectral_ops.py
index 3dc4d72..e71d980 100644
--- a/test/test_spectral_ops.py
+++ b/test/test_spectral_ops.py
@@ -13,7 +13,7 @@
skipIf)
from torch.testing._internal.common_methods_invocations import spectral_funcs
-from distutils.version import LooseVersion
+from setuptools import distutils
from typing import Optional, List
@@ -101,7 +101,7 @@
@ops([op for op in spectral_funcs if not op.ndimensional])
def test_reference_1d(self, device, dtype, op):
norm_modes = ((None, "forward", "backward", "ortho")
- if LooseVersion(np.__version__) >= '1.20.0'
+ if distutils.version.LooseVersion(np.__version__) >= '1.20.0'
else (None, "ortho"))
test_args = [
*product(
@@ -252,7 +252,7 @@
@ops([op for op in spectral_funcs if op.ndimensional])
def test_reference_nd(self, device, dtype, op):
norm_modes = ((None, "forward", "backward", "ortho")
- if LooseVersion(np.__version__) >= '1.20.0'
+ if distutils.version.LooseVersion(np.__version__) >= '1.20.0'
else (None, "ortho"))
# input_ndim, s, dim
@@ -349,7 +349,7 @@
@dtypes(torch.double, torch.complex128)
def test_fft2_numpy(self, device, dtype):
norm_modes = ((None, "forward", "backward", "ortho")
- if LooseVersion(np.__version__) >= '1.20.0'
+ if distutils.version.LooseVersion(np.__version__) >= '1.20.0'
else (None, "ortho"))
# input_ndim, s
diff --git a/tools/build_pytorch_libs.py b/tools/build_pytorch_libs.py
index 2a40c90..a29a7e0 100644
--- a/tools/build_pytorch_libs.py
+++ b/tools/build_pytorch_libs.py
@@ -5,11 +5,11 @@
from .setup_helpers.env import IS_64BIT, IS_WINDOWS, check_negative_env_flag
from .setup_helpers.cmake import USE_NINJA
+from setuptools import distutils
def _overlay_windows_vcvars(env):
- from distutils._msvccompiler import _get_vc_env
vc_arch = 'x64' if IS_64BIT else 'x86'
- vc_env = _get_vc_env(vc_arch)
+ vc_env = distutils._msvccompiler._get_vc_env(vc_arch)
# Keys in `_get_vc_env` are always lowercase.
# We turn them into uppercase before overlaying vcvars
# because OS environ keys are always uppercase on Windows.
diff --git a/tools/generate_torch_version.py b/tools/generate_torch_version.py
index 8dbd9b8..2637e3b 100644
--- a/tools/generate_torch_version.py
+++ b/tools/generate_torch_version.py
@@ -2,7 +2,7 @@
import os
import subprocess
from pathlib import Path
-from distutils.util import strtobool
+from setuptools import distutils
from typing import Optional, Union
def get_sha(pytorch_root: Union[str, Path]) -> str:
@@ -29,7 +29,7 @@
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Generate torch/version.py from build and environment metadata.")
- parser.add_argument("--is_debug", type=strtobool, help="Whether this build is debug mode or not.")
+ parser.add_argument("--is_debug", type=distutils.util.strtobool, help="Whether this build is debug mode or not.")
parser.add_argument("--cuda_version", type=str)
parser.add_argument("--hip_version", type=str)
diff --git a/tools/setup_helpers/cmake.py b/tools/setup_helpers/cmake.py
index 14a1666..9ce09c4 100644
--- a/tools/setup_helpers/cmake.py
+++ b/tools/setup_helpers/cmake.py
@@ -7,8 +7,8 @@
import re
from subprocess import check_call, check_output, CalledProcessError
import sys
-import distutils.sysconfig
-from distutils.version import LooseVersion
+import sysconfig
+from setuptools import distutils
from . import which
from .env import (BUILD_DIR, IS_64BIT, IS_DARWIN, IS_WINDOWS, check_negative_env_flag)
@@ -115,10 +115,10 @@
return cmake_command
cmake3 = which('cmake3')
cmake = which('cmake')
- if cmake3 is not None and CMake._get_version(cmake3) >= LooseVersion("3.5.0"):
+ if cmake3 is not None and CMake._get_version(cmake3) >= distutils.version.LooseVersion("3.5.0"):
cmake_command = 'cmake3'
return cmake_command
- elif cmake is not None and CMake._get_version(cmake) >= LooseVersion("3.5.0"):
+ elif cmake is not None and CMake._get_version(cmake) >= distutils.version.LooseVersion("3.5.0"):
return cmake_command
else:
raise RuntimeError('no cmake or cmake3 with version >= 3.5.0 found')
@@ -129,7 +129,7 @@
for line in check_output([cmd, '--version']).decode('utf-8').split('\n'):
if 'version' in line:
- return LooseVersion(line.strip().split(' ')[2])
+ return distutils.version.LooseVersion(line.strip().split(' ')[2])
raise RuntimeError('no version found')
def run(self, args, env):
@@ -217,7 +217,7 @@
# Store build options that are directly stored in environment variables
build_options = {
# The default value cannot be easily obtained in CMakeLists.txt. We set it here.
- 'CMAKE_PREFIX_PATH': distutils.sysconfig.get_python_lib()
+ 'CMAKE_PREFIX_PATH': sysconfig.get_path('purelib')
}
# Build options that do not start with "BUILD_", "USE_", or "CMAKE_" and are directly controlled by env vars.
# This is a dict that maps environment variables to the corresponding variable name in CMake.
@@ -304,7 +304,7 @@
CMake.defines(args,
PYTHON_EXECUTABLE=sys.executable,
PYTHON_LIBRARY=cmake_python_library,
- PYTHON_INCLUDE_DIR=distutils.sysconfig.get_python_inc(),
+ PYTHON_INCLUDE_DIR=sysconfig.get_path('include'),
TORCH_BUILD_VERSION=version,
NUMPY_INCLUDE_DIR=NUMPY_INCLUDE_DIR,
**build_options)
diff --git a/torch/testing/_internal/common_cuda.py b/torch/testing/_internal/common_cuda.py
index 1681262..30610ec 100644
--- a/torch/testing/_internal/common_cuda.py
+++ b/torch/testing/_internal/common_cuda.py
@@ -6,7 +6,7 @@
from torch.testing._internal.common_utils import TEST_NUMBA
import inspect
import contextlib
-from distutils.version import LooseVersion
+from setuptools import distutils
TEST_CUDA = torch.cuda.is_available()
@@ -16,7 +16,7 @@
TEST_CUDNN = TEST_CUDA and torch.backends.cudnn.is_acceptable(torch.tensor(1., device=CUDA_DEVICE))
TEST_CUDNN_VERSION = torch.backends.cudnn.version() if TEST_CUDNN else 0
-CUDA11OrLater = torch.version.cuda and LooseVersion(torch.version.cuda) >= "11.0.0"
+CUDA11OrLater = torch.version.cuda and distutils.version.LooseVersion(torch.version.cuda) >= "11.0.0"
CUDA9 = torch.version.cuda and torch.version.cuda.startswith('9.')
SM53OrLater = torch.cuda.is_available() and torch.cuda.get_device_capability() >= (5, 3)
diff --git a/torch/testing/_internal/common_methods_invocations.py b/torch/testing/_internal/common_methods_invocations.py
index ee57ae0..4b792f8 100644
--- a/torch/testing/_internal/common_methods_invocations.py
+++ b/torch/testing/_internal/common_methods_invocations.py
@@ -34,7 +34,7 @@
torch_to_numpy_dtype_dict, slowTest, TEST_WITH_ASAN, _wrap_warn_once,
GRADCHECK_NONDET_TOL,)
-from distutils.version import LooseVersion
+from setuptools import distutils
if TEST_SCIPY:
import scipy.special
@@ -5470,11 +5470,11 @@
skips=(
# Reference: https://github.com/pytorch/pytorch/pull/49155#issuecomment-742664611
SkipInfo('TestUnaryUfuncs', 'test_reference_numerics_extremal',
- active_if=TEST_SCIPY and LooseVersion(scipy.__version__) < "1.4.0"),
+ active_if=TEST_SCIPY and distutils.version.LooseVersion(scipy.__version__) < "1.4.0"),
SkipInfo('TestUnaryUfuncs', 'test_reference_numerics_hard',
- active_if=TEST_SCIPY and LooseVersion(scipy.__version__) < "1.4.0"),
+ active_if=TEST_SCIPY and distutils.version.LooseVersion(scipy.__version__) < "1.4.0"),
SkipInfo('TestUnaryUfuncs', 'test_reference_numerics_normal',
- active_if=TEST_SCIPY and LooseVersion(scipy.__version__) < "1.4.0"),
+ active_if=TEST_SCIPY and distutils.version.LooseVersion(scipy.__version__) < "1.4.0"),
)),
UnaryUfuncInfo('lgamma',
ref=reference_lgamma if TEST_SCIPY else _NOTHING,
diff --git a/torch/utils/cpp_extension.py b/torch/utils/cpp_extension.py
index 7c5f2b2..991a99f 100644
--- a/torch/utils/cpp_extension.py
+++ b/torch/utils/cpp_extension.py
@@ -1635,13 +1635,12 @@
env = os.environ.copy()
# Try to activate the vc env for the users
if IS_WINDOWS and 'VSCMD_ARG_TGT_ARCH' not in env:
- from distutils.util import get_platform
- from distutils._msvccompiler import _get_vc_env
+ from setuptools import distutils
- plat_name = get_platform()
+ plat_name = distutils.util.get_platform()
plat_spec = PLAT_TO_VCVARS[plat_name]
- vc_env = _get_vc_env(plat_spec)
+ vc_env = distutils._msvccompiler._get_vc_env(plat_spec)
vc_env = {k.upper(): v for k, v in vc_env.items()}
for k, v in env.items():
uk = k.upper()
diff --git a/torch/utils/tensorboard/__init__.py b/torch/utils/tensorboard/__init__.py
index 32bc82f..bb1885d 100644
--- a/torch/utils/tensorboard/__init__.py
+++ b/torch/utils/tensorboard/__init__.py
@@ -1,7 +1,12 @@
import tensorboard
-from distutils.version import LooseVersion
+from setuptools import distutils
+
+LooseVersion = distutils.version.LooseVersion
+
if not hasattr(tensorboard, '__version__') or LooseVersion(tensorboard.__version__) < LooseVersion('1.15'):
raise ImportError('TensorBoard logging requires TensorBoard version 1.15 or above')
+
+del distutils
del LooseVersion
del tensorboard
