Remove distutils (pytorch#57040)

Summary: [distutils](https://docs.python.org/3/library/distutils.html) is on its way out and will be deprecated-on-import for Python 3.10+ and removed in Python 3.12 (see [PEP 632](https://www.python.org/dev/peps/pep-0632/)). There's no reason for us to keep it around since all the functionality we want from it can be found in `setuptools` / `sysconfig`. `setuptools` includes a copy of most of `distutils` (which is fine to use according to the PEP), that it uses under the hood, so this PR also uses that in some places. Fixes pytorch#56527 Pull Request resolved: pytorch#57040 Pulled By: driazati Reviewed By: nikithamalgifb Differential Revision: D28051356 fbshipit-source-id: 1ca312219032540e755593e50da0c9e23c62d720
krshrimali · May 18, 2021 · 361bce4 · 361bce4
1 parent 84f3bb4
commit 361bce4
Show file tree

Hide file tree

Showing 15 changed files with 94 additions and 82 deletions.
diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
@@ -1706,8 +1706,9 @@ if(BUILD_PYTHON)
   # https://github.com/pytorch/pytorch/tree/master/tools/build_pytorch_libs.bat#note-backslash-munging-on-windows
   pycmd(PYTHON_SITE_PACKAGES "
       import os
-      from distutils import sysconfig
-      print(sysconfig.get_python_lib(prefix=''))
+      import sysconfig
+      relative_site_packages = sysconfig.get_path('purelib').replace(sysconfig.get_path('data'), '').lstrip(os.path.sep)
+      print(relative_site_packages)
   ")
   file(TO_CMAKE_PATH ${PYTHON_SITE_PACKAGES} PYTHON_SITE_PACKAGES)
   set(PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES} PARENT_SCOPE) # for Summary
@@ -1718,9 +1719,21 @@ if(BUILD_PYTHON)
   # Try to get from python through sysconfig.get_env_var('EXT_SUFFIX') first,
   # fallback to ".pyd" if windows and ".so" for all others.
   pycmd(PY_EXT_SUFFIX "
-      from distutils import sysconfig
-      ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
-      print(ext_suffix if ext_suffix else '')
+      def get_ext_suffix():
+          import sys
+          if sys.version_info < (3, 8) and sys.platform == 'win32':
+              # Workaround for https://bugs.python.org/issue39825
+              import _imp
+              return _imp.extension_suffixes()[0]
+          else:
+              import sysconfig
+              return sysconfig.get_config_var('EXT_SUFFIX')
+
+      suffix = get_ext_suffix()
+      if suffix is not None:
+          print(suffix)
+      else:
+          print()
   ")
   if("${PY_EXT_SUFFIX}" STREQUAL "")
     if(MSVC)

diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake
@@ -935,23 +935,18 @@ if(BUILD_PYTHON)
   # then these will just use "python", but at least they'll be consistent with
   # each other).
   if(NOT DEFINED PYTHON_INCLUDE_DIR)
-    # distutils.sysconfig, if it's installed, is more accurate than sysconfig,
-    # which sometimes outputs directories that do not exist
-    pycmd_no_exit(_py_inc _exitcode "from distutils import sysconfig; print(sysconfig.get_python_inc())")
+    # TODO: Verify that sysconfig isn't inaccurate
+    pycmd_no_exit(_py_inc _exitcode "import sysconfig; print(sysconfig.get_path('include'))")
     if("${_exitcode}" EQUAL 0 AND IS_DIRECTORY "${_py_inc}")
       set(PYTHON_INCLUDE_DIR "${_py_inc}")
-      message(STATUS "Setting Python's include dir to ${_py_inc} from distutils.sysconfig")
+      message(STATUS "Setting Python's include dir to ${_py_inc} from sysconfig")
     else()
-      pycmd_no_exit(_py_inc _exitcode "from sysconfig import get_paths; print(get_paths()['include'])")
-      if("${_exitcode}" EQUAL 0 AND IS_DIRECTORY "${_py_inc}")
-        set(PYTHON_INCLUDE_DIR "${_py_inc}")
-        message(STATUS "Setting Python's include dir to ${_py_inc} from sysconfig")
-      endif()
+      message(WARNING "Could not set Python's include dir to ${_py_inc} from sysconfig")
     endif()
   endif(NOT DEFINED PYTHON_INCLUDE_DIR)
 
   if(NOT DEFINED PYTHON_LIBRARY)
-    pycmd_no_exit(_py_lib _exitcode "from sysconfig import get_paths; print(get_paths()['stdlib'])")
+    pycmd_no_exit(_py_lib _exitcode "import sysconfig; print(sysconfig.get_path('stdlib'))")
     if("${_exitcode}" EQUAL 0 AND EXISTS "${_py_lib}" AND EXISTS "${_py_lib}")
       set(PYTHON_LIBRARY "${_py_lib}")
       if(MSVC)

diff --git a/scripts/build_android.sh b/scripts/build_android.sh
@@ -61,7 +61,7 @@ CMAKE_ARGS=()
 
 if [ -z "${BUILD_CAFFE2_MOBILE:-}" ]; then
   # Build PyTorch mobile
-  CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
+  CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
   CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
   CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
   # custom build with selected ops

diff --git a/scripts/build_ios.sh b/scripts/build_ios.sh
@@ -13,7 +13,7 @@ CMAKE_ARGS=()
 
 if [ -z "${BUILD_CAFFE2_MOBILE:-}" ]; then
   # Build PyTorch mobile
-  CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
+  CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
   CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
   CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
   # custom build with selected ops

diff --git a/scripts/build_mobile.sh b/scripts/build_mobile.sh
@@ -15,7 +15,7 @@ echo "Bash: $(/bin/bash --version | head -1)"
 echo "Caffe2 path: $CAFFE2_ROOT"
 
 CMAKE_ARGS=()
-CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')")
+CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
 CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
 CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
 CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF")

diff --git a/scripts/get_python_cmake_flags.py b/scripts/get_python_cmake_flags.py
@@ -15,12 +15,12 @@
 
 
 
-from distutils import sysconfig
+import sysconfig
 import sys
 
 flags = [
     '-DPYTHON_EXECUTABLE:FILEPATH={}'.format(sys.executable),
-    '-DPYTHON_INCLUDE_DIR={}'.format(sysconfig.get_python_inc()),
+    '-DPYTHON_INCLUDE_DIR={}'.format(sysconfig.get_path('include')),
 ]
 
 print(' '.join(flags), end='')
diff --git a/setup.py b/setup.py
@@ -21,9 +21,7 @@
 #     default behavior of autogoo and cmake build systems.)
 #
 #   CC
-#     the C/C++ compiler to use (NB: the CXX flag has no effect for distutils
-#     compiles, because distutils always uses CC to compile, even for C++
-#     files.
+#     the C/C++ compiler to use
 #
 # Environment variables for feature toggles:
 #
@@ -197,14 +195,10 @@
 
 from setuptools import setup, Extension, find_packages
 from collections import defaultdict
-from distutils import core
-from distutils.core import Distribution
-from distutils.errors import DistutilsArgError
+from setuptools.dist import Distribution
 import setuptools.command.build_ext
 import setuptools.command.install
-import distutils.command.clean
-import distutils.command.sdist
-import distutils.sysconfig
+import setuptools.command.sdist
 import filecmp
 import shutil
 import subprocess
@@ -213,6 +207,7 @@
 import glob
 import importlib
 import time
+import sysconfig
 
 from tools.build_pytorch_libs import build_caffe2
 from tools.setup_helpers.env import (IS_WINDOWS, IS_DARWIN, IS_LINUX,
@@ -261,31 +256,31 @@ def report(*args):
     def report(*args):
         pass
 
+    # Make distutils respect --quiet too
+    setuptools.distutils.log.warn = report
+
 # Constant known variables used throughout this file
 cwd = os.path.dirname(os.path.abspath(__file__))
 lib_path = os.path.join(cwd, "torch", "lib")
 third_party_path = os.path.join(cwd, "third_party")
 caffe2_build_dir = os.path.join(cwd, "build")
-# lib/pythonx.x/site-packages
-rel_site_packages = distutils.sysconfig.get_python_lib(prefix='')
-# full absolute path to the dir above
-full_site_packages = distutils.sysconfig.get_python_lib()
+
 # CMAKE: full path to python library
 if IS_WINDOWS:
     cmake_python_library = "{}/libs/python{}.lib".format(
-        distutils.sysconfig.get_config_var("prefix"),
-        distutils.sysconfig.get_config_var("VERSION"))
+        sysconfig.get_config_var("prefix"),
+        sysconfig.get_config_var("VERSION"))
     # Fix virtualenv builds
     # TODO: Fix for python < 3.3
     if not os.path.exists(cmake_python_library):
         cmake_python_library = "{}/libs/python{}.lib".format(
             sys.base_prefix,
-            distutils.sysconfig.get_config_var("VERSION"))
+            sysconfig.get_config_var("VERSION"))
 else:
     cmake_python_library = "{}/{}".format(
-        distutils.sysconfig.get_config_var("LIBDIR"),
-        distutils.sysconfig.get_config_var("INSTSONAME"))
-cmake_python_include_dir = distutils.sysconfig.get_python_inc()
+        sysconfig.get_config_var("LIBDIR"),
+        sysconfig.get_config_var("INSTSONAME"))
+cmake_python_include_dir = sysconfig.get_path("include")
 
 
 ################################################################################
@@ -491,9 +486,9 @@ def run(self):
 
         # Do not use clang to compile extensions if `-fstack-clash-protection` is defined
         # in system CFLAGS
-        system_c_flags = str(distutils.sysconfig.get_config_var('CFLAGS'))
-        if IS_LINUX and '-fstack-clash-protection' in system_c_flags and 'clang' in os.environ.get('CC', ''):
-            os.environ['CC'] = str(distutils.sysconfig.get_config_var('CC'))
+        c_flags = str(os.getenv('CFLAGS', ''))
+        if IS_LINUX and '-fstack-clash-protection' in c_flags and 'clang' in os.environ.get('CC', ''):
+            os.environ['CC'] = str(os.environ['CC'])
 
         # It's an old-style class in Python 2.7...
         setuptools.command.build_ext.build_ext.run(self)
@@ -547,7 +542,8 @@ def build_extensions(self):
             filename = self.get_ext_filename(fullname)
             report("\nCopying extension {}".format(ext.name))
 
-            src = os.path.join("torch", rel_site_packages, filename)
+            relative_site_packages = sysconfig.get_path('purelib').replace(sysconfig.get_path('data'), '').lstrip(os.path.sep)
+            src = os.path.join("torch", relative_site_packages, filename)
             if not os.path.exists(src):
                 report("{} does not exist".format(src))
                 del self.extensions[i]
@@ -559,10 +555,11 @@ def build_extensions(self):
                     os.makedirs(dst_dir)
                 self.copy_file(src, dst)
                 i += 1
-        distutils.command.build_ext.build_ext.build_extensions(self)
+        setuptools.command.build_ext.build_ext.build_extensions(self)
+
 
     def get_outputs(self):
-        outputs = distutils.command.build_ext.build_ext.get_outputs(self)
+        outputs = setuptools.command.build_ext.build_ext.get_outputs(self)
         outputs.append(os.path.join(self.build_lib, "caffe2"))
         report("setup.py::get_outputs returning {}".format(outputs))
         return outputs
@@ -644,7 +641,15 @@ def run(self):
         super().run()
 
 
-class clean(distutils.command.clean.clean):
+class clean(setuptools.Command):
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
     def run(self):
         import glob
         import re
@@ -665,10 +670,8 @@ def run(self):
                         except OSError:
                             shutil.rmtree(filename, ignore_errors=True)
 
-        super().run()
 
-
-class sdist(distutils.command.sdist.sdist):
+class sdist(setuptools.command.sdist.sdist):
     def run(self):
         with concat_license_files():
             super().run()
@@ -863,14 +866,11 @@ def print_box(msg):
     # Parse the command line and check the arguments
     # before we proceed with building deps and setup
     dist = Distribution()
-    dist.script_name = sys.argv[0]
-    dist.script_args = sys.argv[1:]
     try:
-        ok = dist.parse_command_line()
-    except DistutilsArgError as msg:
-        raise SystemExit(core.gen_usage(dist.script_name) + "\nerror: %s" % msg)
-    if not ok:
-        sys.exit()
+        dist.parse_command_line()
+    except setuptools.distutils.errors.DistutilsArgError as e:
+        print(e)
+        sys.exit(1)
 
     if RUN_BUILD_DEPS:
         build_deps()

diff --git a/test/test_spectral_ops.py b/test/test_spectral_ops.py
@@ -13,7 +13,7 @@
      skipIf)
 from torch.testing._internal.common_methods_invocations import spectral_funcs
 
-from distutils.version import LooseVersion
+from setuptools import distutils
 from typing import Optional, List
 
 
@@ -101,7 +101,7 @@ class TestFFT(TestCase):
     @ops([op for op in spectral_funcs if not op.ndimensional])
     def test_reference_1d(self, device, dtype, op):
         norm_modes = ((None, "forward", "backward", "ortho")
-                      if LooseVersion(np.__version__) >= '1.20.0'
+                      if distutils.version.LooseVersion(np.__version__) >= '1.20.0'
                       else (None, "ortho"))
         test_args = [
             *product(
@@ -252,7 +252,7 @@ def test_fft_half_and_bfloat16_errors(self, device, dtype, op):
     @ops([op for op in spectral_funcs if op.ndimensional])
     def test_reference_nd(self, device, dtype, op):
         norm_modes = ((None, "forward", "backward", "ortho")
-                      if LooseVersion(np.__version__) >= '1.20.0'
+                      if distutils.version.LooseVersion(np.__version__) >= '1.20.0'
                       else (None, "ortho"))
 
         # input_ndim, s, dim
@@ -349,7 +349,7 @@ def test_fftn_invalid(self, device, dtype, op):
     @dtypes(torch.double, torch.complex128)
     def test_fft2_numpy(self, device, dtype):
         norm_modes = ((None, "forward", "backward", "ortho")
-                      if LooseVersion(np.__version__) >= '1.20.0'
+                      if distutils.version.LooseVersion(np.__version__) >= '1.20.0'
                       else (None, "ortho"))
 
         # input_ndim, s

diff --git a/tools/build_pytorch_libs.py b/tools/build_pytorch_libs.py
@@ -5,11 +5,11 @@
 from .setup_helpers.env import IS_64BIT, IS_WINDOWS, check_negative_env_flag
 from .setup_helpers.cmake import USE_NINJA
 
+from setuptools import distutils
 
 def _overlay_windows_vcvars(env):
-    from distutils._msvccompiler import _get_vc_env
     vc_arch = 'x64' if IS_64BIT else 'x86'
-    vc_env = _get_vc_env(vc_arch)
+    vc_env = distutils._msvccompiler._get_vc_env(vc_arch)
     # Keys in `_get_vc_env` are always lowercase.
     # We turn them into uppercase before overlaying vcvars
     # because OS environ keys are always uppercase on Windows.

diff --git a/tools/generate_torch_version.py b/tools/generate_torch_version.py
@@ -2,7 +2,7 @@
 import os
 import subprocess
 from pathlib import Path
-from distutils.util import strtobool
+from setuptools import distutils
 from typing import Optional, Union
 
 def get_sha(pytorch_root: Union[str, Path]) -> str:
@@ -29,7 +29,7 @@ def get_torch_version(sha: Optional[str] = None) -> str:
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Generate torch/version.py from build and environment metadata.")
-    parser.add_argument("--is_debug", type=strtobool, help="Whether this build is debug mode or not.")
+    parser.add_argument("--is_debug", type=distutils.util.strtobool, help="Whether this build is debug mode or not.")
     parser.add_argument("--cuda_version", type=str)
     parser.add_argument("--hip_version", type=str)
 

diff --git a/tools/setup_helpers/cmake.py b/tools/setup_helpers/cmake.py
@@ -7,8 +7,8 @@
 import re
 from subprocess import check_call, check_output, CalledProcessError
 import sys
-import distutils.sysconfig
-from distutils.version import LooseVersion
+import sysconfig
+from setuptools import distutils
 
 from . import which
 from .env import (BUILD_DIR, IS_64BIT, IS_DARWIN, IS_WINDOWS, check_negative_env_flag)
@@ -115,10 +115,10 @@ def _get_cmake_command():
             return cmake_command
         cmake3 = which('cmake3')
         cmake = which('cmake')
-        if cmake3 is not None and CMake._get_version(cmake3) >= LooseVersion("3.5.0"):
+        if cmake3 is not None and CMake._get_version(cmake3) >= distutils.version.LooseVersion("3.5.0"):
             cmake_command = 'cmake3'
             return cmake_command
-        elif cmake is not None and CMake._get_version(cmake) >= LooseVersion("3.5.0"):
+        elif cmake is not None and CMake._get_version(cmake) >= distutils.version.LooseVersion("3.5.0"):
             return cmake_command
         else:
             raise RuntimeError('no cmake or cmake3 with version >= 3.5.0 found')
@@ -129,7 +129,7 @@ def _get_version(cmd):
 
         for line in check_output([cmd, '--version']).decode('utf-8').split('\n'):
             if 'version' in line:
-                return LooseVersion(line.strip().split(' ')[2])
+                return distutils.version.LooseVersion(line.strip().split(' ')[2])
         raise RuntimeError('no version found')
 
     def run(self, args, env):
@@ -217,7 +217,7 @@ def generate(self, version, cmake_python_library, build_python, build_test, my_e
         # Store build options that are directly stored in environment variables
         build_options = {
             # The default value cannot be easily obtained in CMakeLists.txt. We set it here.
-            'CMAKE_PREFIX_PATH': distutils.sysconfig.get_python_lib()
+            'CMAKE_PREFIX_PATH': sysconfig.get_path('purelib')
         }
         # Build options that do not start with "BUILD_", "USE_", or "CMAKE_" and are directly controlled by env vars.
         # This is a dict that maps environment variables to the corresponding variable name in CMake.
@@ -304,7 +304,7 @@ def generate(self, version, cmake_python_library, build_python, build_test, my_e
         CMake.defines(args,
                       PYTHON_EXECUTABLE=sys.executable,
                       PYTHON_LIBRARY=cmake_python_library,
-                      PYTHON_INCLUDE_DIR=distutils.sysconfig.get_python_inc(),
+                      PYTHON_INCLUDE_DIR=sysconfig.get_path('include'),
                       TORCH_BUILD_VERSION=version,
                       NUMPY_INCLUDE_DIR=NUMPY_INCLUDE_DIR,
                       **build_options)