Switch install scheme backend to sysconfig

This is only done for Python 3.10+ so we don't disrupt too many existing
pip installations.

news/10358.removal.rst

@@ -0,0 +1,4 @@
+On Python 3.10 or later, the installation scheme backend has been changed to use
+``sysconfig``. This is to anticipate the deprecation of ``distutils`` in Python
+3.10, and its scheduled removal in 3.12. For compatibility considerations, pip
+installations running on Python 3.9 or lower will continue to use ``distutils``.

setup.cfg

@@ -57,6 +57,7 @@ follow_imports = skip
 addopts = --ignore src/pip/_vendor --ignore tests/tests_cache -r aR --color=yes
 markers =
     network: tests that need network
+    incompatible_with_sysconfig
     incompatible_with_test_venv
     incompatible_with_venv
     no_auto_tempdir_manager

src/pip/_internal/locations/__init__.py

@@ -45,6 +45,8 @@
 
 _PLATLIBDIR: str = getattr(sys, "platlibdir", "lib")
 
+_USE_SYSCONFIG = sys.version_info >= (3, 10)
+
 
 def _looks_like_bpo_44860() -> bool:
     """The resolution to bpo-44860 will change this incorrect platlib.
@@ -190,15 +192,18 @@ def get_scheme(
     isolated: bool = False,
     prefix: Optional[str] = None,
 ) -> Scheme:
-    old = _distutils.get_scheme(
+    new = _sysconfig.get_scheme(
         dist_name,
         user=user,
         home=home,
         root=root,
         isolated=isolated,
         prefix=prefix,
     )
-    new = _sysconfig.get_scheme(
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_scheme(
         dist_name,
         user=user,
         home=home,
@@ -335,8 +340,11 @@ def get_scheme(
 
 
 def get_bin_prefix() -> str:
-    old = _distutils.get_bin_prefix()
     new = _sysconfig.get_bin_prefix()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_bin_prefix()
     if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix"):
         _log_context()
     return old
@@ -365,8 +373,11 @@ def _looks_like_deb_system_dist_packages(value: str) -> bool:
 
 def get_purelib() -> str:
     """Return the default pure-Python lib location."""
-    old = _distutils.get_purelib()
     new = _sysconfig.get_purelib()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_purelib()
     if _looks_like_deb_system_dist_packages(old):
         return old
     if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="purelib"):
@@ -376,19 +387,32 @@ def get_purelib() -> str:
 
 def get_platlib() -> str:
     """Return the default platform-shared lib location."""
-    old = _distutils.get_platlib()
     new = _sysconfig.get_platlib()
+    if _USE_SYSCONFIG:
+        return new
+
+    old = _distutils.get_platlib()
     if _looks_like_deb_system_dist_packages(old):
         return old
     if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="platlib"):
         _log_context()
     return old
 
 
+def _deduplicated(v1: str, v2: str) -> List[str]:
+    """Deduplicate values from a list."""
+    if v1 == v2:
+        return [v1]
+    return [v1, v2]
+
+
 def get_prefixed_libs(prefix: str) -> List[str]:
     """Return the lib locations under ``prefix``."""
-    old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
     new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix)
+    if _USE_SYSCONFIG:
+        return _deduplicated(new_pure, new_plat)
+
+    old_pure, old_plat = _distutils.get_prefixed_libs(prefix)
 
     warned = [
         _warn_if_mismatch(
@@ -405,6 +429,4 @@ def get_prefixed_libs(prefix: str) -> List[str]:
     if any(warned):
         _log_context(prefix=prefix)
 
-    if old_pure == old_plat:
-        return [old_pure]
-    return [old_pure, old_plat]
+    return _deduplicated(old_pure, old_plat)

src/pip/_internal/operations/install/wheel.py

@@ -628,10 +628,7 @@ def pyc_output_path(path: str) -> str:
             with warnings.catch_warnings():
                 warnings.filterwarnings("ignore")
                 for path in pyc_source_file_paths():
-                    # Python 2's `compileall.compile_file` requires a str in
-                    # error cases, so we must convert to the native type.
-                    path_arg = ensure_str(path, encoding=sys.getfilesystemencoding())
-                    success = compileall.compile_file(path_arg, force=True, quiet=True)
+                    success = compileall.compile_file(path, force=True, quiet=True)
                     if success:
                         pyc_path = pyc_output_path(path)
                         assert os.path.exists(pyc_path)

tests/conftest.py

@@ -15,6 +15,7 @@
 from setuptools.wheel import Wheel
 
 from pip._internal.cli.main import main as pip_entry_point
+from pip._internal.locations import _USE_SYSCONFIG
 from pip._internal.utils.temp_dir import global_tempdir_manager
 from tests.lib import DATA_DIR, SRC_DIR, PipTestEnvironment, TestData
 from tests.lib.certs import make_tls_cert, serialize_cert, serialize_key
@@ -77,6 +78,9 @@ def pytest_collection_modifyitems(config, items):
         ):
             item.add_marker(pytest.mark.skip("Incompatible with venv"))
 
+        if item.get_closest_marker("incompatible_with_sysconfig") and _USE_SYSCONFIG:
+            item.add_marker(pytest.mark.skip("Incompatible with sysconfig"))
+
         module_path = os.path.relpath(
             item.module.__file__,
             os.path.commonprefix([__file__, item.module.__file__]),

tests/unit/test_locations.py

@@ -96,6 +96,7 @@ def test_root_modifies_appropriately(self, monkeypatch):
             expected = os.path.join(root, path[1:])
             assert os.path.abspath(root_scheme[key]) == expected
 
+    @pytest.mark.incompatible_with_sysconfig
     @pytest.mark.incompatible_with_venv
     def test_distutils_config_file_read(self, tmpdir, monkeypatch):
         # This deals with nt/posix path differences
@@ -116,10 +117,11 @@ def test_distutils_config_file_read(self, tmpdir, monkeypatch):
         scheme = _get_scheme_dict("example")
         assert scheme["scripts"] == install_scripts
 
+    @pytest.mark.incompatible_with_sysconfig
     @pytest.mark.incompatible_with_venv
     # when we request install-lib, we should install everything (.py &
     # .so) into that path; i.e. ensure platlib & purelib are set to
-    # this path
+    # this path. sysconfig does not support this.
     def test_install_lib_takes_precedence(self, tmpdir, monkeypatch):
         # This deals with nt/posix path differences
         install_lib = os.path.normcase(