3.12 support and no setuptools/wheel on 3.12+ (pypa#2558)

Author: chrysle

.github/workflows/check.yml

@@ -15,6 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         py:
+          - "3.12.0-alpha.7"
           - "3.11"
         os:
           - ubuntu-22.04

docs/changelog/2487.feature.rst

@@ -0,0 +1,6 @@
+Do not install ``wheel`` and ``setuptools`` seed packages for Python 3.12+. To restore the old behaviour use:
+
+- for ``wheel`` use ``VIRTUALENV_WHEEL=bundle`` environment variable or ``--wheel=bundle`` CLI flag,
+- for ``setuptools`` use ``VIRTUALENV_SETUPTOOLS=bundle`` environment variable or ``--setuptools=bundle`` CLI flag.
+
+By :user:`chrysle`.

docs/changelog/2558.feature.rst

@@ -0,0 +1 @@
+3.12 support - by :user:`gaborbernat`.

docs/render_cli.py

@@ -180,15 +180,7 @@ def _get_help_text(row):
             content = row.help[: row.help.index("(") - 1]
         else:
             content = row.help
-        if name in ("--setuptools", "--pip", "--wheel"):
-            text = row.help
-            at = text.index(" bundle ")
-            help_body = n.paragraph("")
-            help_body += n.Text(text[: at + 1])
-            help_body += n.literal(text="bundle")
-            help_body += n.Text(text[at + 7 :])
-        else:
-            help_body = n.paragraph("", "", n.Text(content))
+        help_body = n.paragraph("", "", n.Text(content))
         if row.choices is not None:
             help_body += n.Text("; choice of: ")
             first = True

docs/user_guide.rst

@@ -118,8 +118,9 @@ at the moment has two types of virtual environments:
 Seeders
 -------
 These will install for you some seed packages (one or more of: :pypi:`pip`, :pypi:`setuptools`, :pypi:`wheel`) that
-enables you to install additional python packages into the created virtual environment (by invoking pip). There are two
-main seed mechanism available:
+enables you to install additional python packages into the created virtual environment (by invoking pip). Installing
+:pypi:`setuptools` and :pypi:`wheel` is disabled by default on Python 3.12+ environments. There are two
+main seed mechanisms available:
 
 - ``pip`` - this method uses the bundled pip with virtualenv to install the seed packages (note, a new child process
   needs to be created to do this, which can be expensive especially on Windows).

pyproject.toml

@@ -59,10 +59,12 @@ optional-dependencies.test = [
   "packaging>=23.1",
   "pytest>=7.3.1",
   "pytest-env>=0.8.1",
-  "pytest-freezegun>=0.4.2",
+  'pytest-freezegun>=0.4.2; platform_python_implementation == "PyPy"',
   "pytest-mock>=3.10",
   "pytest-randomly>=3.12",
   "pytest-timeout>=2.1",
+  "setuptools>=67.7.1",
+  'time-machine>=2.9; platform_python_implementation == "CPython"',
 ]
 urls.Documentation = "https://virtualenv.pypa.io"
 urls.Homepage = "https://github.com/pypa/virtualenv"
@@ -119,7 +121,7 @@ ignore = [
 [tool.pytest.ini_options]
 markers = ["slow"]
 timeout = 600
-addopts = "--tb=auto -ra --showlocals --no-success-flaky-report"
+addopts = "--showlocals --no-success-flaky-report"
 env = ["PYTHONIOENCODING=utf-8"]
 tmnet_api_key = "virtualenv_Puh0iZYgE22xQPgMOft5hTO3eR8AG7cwZb2DedvykbBqevWC9QivZUot1JkTFWom"
 testmon_ignore_dependencies = ["virtualenv",]

src/virtualenv/activation/python/__init__.py

@@ -13,9 +13,10 @@ def templates(self):
     def replacements(self, creator, dest_folder):
         replacements = super().replacements(creator, dest_folder)
         lib_folders = OrderedDict((os.path.relpath(str(i), str(dest_folder)), None) for i in creator.libs)
+        lib_folders = os.pathsep.join(lib_folders.keys()).replace("\\", "\\\\")  # escape Windows path characters
         replacements.update(
             {
-                "__LIB_FOLDERS__": os.pathsep.join(lib_folders.keys()),
+                "__LIB_FOLDERS__": lib_folders,
                 "__DECODE_PATH__": "",
             },
         )

src/virtualenv/seed/embed/base_embed.py

@@ -30,18 +30,18 @@ def __init__(self, options):
             self.enabled = False
 
     @classmethod
-    def distributions(cls):
+    def distributions(cls) -> dict[str, Version]:
         return {
             "pip": Version.bundle,
             "setuptools": Version.bundle,
             "wheel": Version.bundle,
         }
 
-    def distribution_to_versions(self):
+    def distribution_to_versions(self) -> dict[str, str]:
         return {
             distribution: getattr(self, f"{distribution}_version")
             for distribution in self.distributions()
-            if getattr(self, f"no_{distribution}") is False
+            if getattr(self, f"no_{distribution}") is False and getattr(self, f"{distribution}_version") != "none"
         }
 
     @classmethod
@@ -71,11 +71,13 @@ def add_parser_arguments(cls, parser, interpreter, app_data):  # noqa: U100
             default=[],
         )
         for distribution, default in cls.distributions().items():
+            if interpreter.version_info[:2] >= (3, 12) and distribution in {"wheel", "setuptools"}:
+                default = "none"
             parser.add_argument(
                 f"--{distribution}",
                 dest=distribution,
                 metavar="version",
-                help=f"version of {distribution} to install as seed: embed, bundle or exact version",
+                help=f"version of {distribution} to install as seed: embed, bundle, none or exact version",
                 default=default,
             )
         for distribution in cls.distributions():
@@ -94,7 +96,7 @@ def add_parser_arguments(cls, parser, interpreter, app_data):  # noqa: U100
             default=not PERIODIC_UPDATE_ON_BY_DEFAULT,
         )
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         result = self.__class__.__name__
         result += "("
         if self.extra_search_dir:
@@ -103,7 +105,10 @@ def __repr__(self):
         for distribution in self.distributions():
             if getattr(self, f"no_{distribution}"):
                 continue
-            ver = f"={getattr(self, f'{distribution}_version', None) or 'latest'}"
+            version = getattr(self, f"{distribution}_version", None)
+            if version == "none":
+                continue
+            ver = f"={version or 'latest'}"
             result += f" {distribution}{ver},"
         return result[:-1] + ")"
 

src/virtualenv/util/path/_sync.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import shutil
+import sys
 from stat import S_IWUSR
 
 
@@ -58,7 +59,8 @@ def onerror(func, path, exc_info):  # noqa: U100
         else:
             raise
 
-    shutil.rmtree(str(dest), ignore_errors=True, onerror=onerror)
+    kwargs = {"onexc" if sys.version_info >= (3, 12) else "onerror": onerror}
+    shutil.rmtree(str(dest), ignore_errors=True, **kwargs)
 
 
 class _Debug:

tests/conftest.py

@@ -12,7 +12,7 @@
 
 from virtualenv.app_data import AppDataDiskFolder
 from virtualenv.discovery.py_info import PythonInfo
-from virtualenv.info import IS_WIN, fs_supports_symlink
+from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink
 from virtualenv.report import LOGGER
 
 
@@ -144,22 +144,6 @@ def _ignore_global_config(tmp_path_factory):
         yield
 
 
-@pytest.fixture(autouse=True, scope="session")
-def _pip_cert(tmp_path_factory):
-    # workaround for https://github.com/pypa/pip/issues/8984 - if the certificate is explicitly set no error can happen
-    key = "PIP_CERT"
-    if key in os.environ:
-        yield
-    else:
-        cert = tmp_path_factory.mktemp("folder") / "cert"
-        import pkgutil
-
-        cert_data = pkgutil.get_data("pip._vendor.certifi", "cacert.pem")
-        cert.write_bytes(cert_data)
-        with change_os_environ(key, str(cert)):
-            yield
-
-
 @pytest.fixture(autouse=True)
 def _check_os_environ_stable():
     old = os.environ.copy()
@@ -368,3 +352,16 @@ def _skip_if_test_in_system(session_app_data):
     current = PythonInfo.current(session_app_data)
     if current.system_executable is not None:
         pytest.skip("test not valid if run under system")
+
+
+if IS_PYPY:
+
+    @pytest.fixture()
+    def time_freeze(freezer):
+        return freezer.move_to
+
+else:
+
+    @pytest.fixture()
+    def time_freeze(time_machine):
+        return lambda s: time_machine.move_to(s, tick=False)

tests/integration/test_run_int.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 import pytest
 
 from virtualenv import cli_run
@@ -8,8 +10,8 @@
 
 
 @pytest.mark.skipif(IS_PYPY, reason="setuptools distutils patching does not work")
-def test_app_data_pinning(tmp_path):
-    version = "19.3.1"
+def test_app_data_pinning(tmp_path: Path) -> None:
+    version = "23.0"
     result = cli_run([str(tmp_path), "--pip", version, "--activators", "", "--seeder", "app-data"])
     code, out, _ = run_cmd([str(result.creator.script("pip")), "list", "--disable-pip-version-check"])
     assert not code

tests/unit/config/test___main__.py

@@ -2,6 +2,7 @@
 
 import re
 import sys
+from pathlib import Path
 from subprocess import PIPE, Popen, check_output
 
 import pytest
@@ -59,8 +60,8 @@ def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys):
 
 
 @pytest.mark.usefixtures("session_app_data")
-def test_session_report_full(tmp_path, capsys):
-    run_with_catch([str(tmp_path)])
+def test_session_report_full(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+    run_with_catch([str(tmp_path), "--setuptools", "bundle", "--wheel", "bundle"])
     out, err = capsys.readouterr()
     assert err == ""
     lines = out.splitlines()

tests/unit/create/test_creator.py

@@ -364,7 +364,19 @@ def test_create_long_path(tmp_path):
 @pytest.mark.parametrize("creator", sorted(set(PythonInfo.current_system().creators().key_to_class) - {"builtin"}))
 @pytest.mark.usefixtures("session_app_data")
 def test_create_distutils_cfg(creator, tmp_path, monkeypatch):
-    result = cli_run([str(tmp_path / "venv"), "--activators", "", "--creator", creator])
+    result = cli_run(
+        [
+            str(tmp_path / "venv"),
+            "--activators",
+            "",
+            "--creator",
+            creator,
+            "--setuptools",
+            "bundle",
+            "--wheel",
+            "bundle",
+        ],
+    )
 
     app = Path(__file__).parent / "console_app"
     dest = tmp_path / "console_app"
@@ -417,7 +429,9 @@ def list_files(path):
 
 def test_zip_importer_can_import_setuptools(tmp_path):
     """We're patching the loaders so might fail on r/o loaders, such as zipimporter on CPython<3.8"""
-    result = cli_run([str(tmp_path / "venv"), "--activators", "", "--no-pip", "--no-wheel", "--copies"])
+    result = cli_run(
+        [str(tmp_path / "venv"), "--activators", "", "--no-pip", "--no-wheel", "--copies", "--setuptools", "bundle"],
+    )
     zip_path = tmp_path / "site-packages.zip"
     with zipfile.ZipFile(str(zip_path), "w", zipfile.ZIP_DEFLATED) as zip_handler:
         lib = str(result.creator.purelib)
@@ -451,6 +465,7 @@ def test_no_preimport_threading(tmp_path):
     out = subprocess.check_output(
         [str(session.creator.exe), "-c", r"import sys; print('\n'.join(sorted(sys.modules)))"],
         text=True,
+        encoding="utf-8",
     )
     imported = set(out.splitlines())
     assert "threading" not in imported
@@ -467,6 +482,7 @@ def test_pth_in_site_vs_python_path(tmp_path):
     out = subprocess.check_output(
         [str(session.creator.exe), "-c", r"import sys; print(sys.testpth)"],
         text=True,
+        encoding="utf-8",
     )
     assert out == "ok\n"
     # same with $PYTHONPATH pointing to site_packages
@@ -479,6 +495,7 @@ def test_pth_in_site_vs_python_path(tmp_path):
         [str(session.creator.exe), "-c", r"import sys; print(sys.testpth)"],
         text=True,
         env=env,
+        encoding="utf-8",
     )
     assert out == "ok\n"
 
@@ -492,6 +509,7 @@ def test_getsitepackages_system_site(tmp_path):
     out = subprocess.check_output(
         [str(session.creator.exe), "-c", r"import site; print(site.getsitepackages())"],
         text=True,
+        encoding="utf-8",
     )
     site_packages = ast.literal_eval(out)
 
@@ -506,6 +524,7 @@ def test_getsitepackages_system_site(tmp_path):
     out = subprocess.check_output(
         [str(session.creator.exe), "-c", r"import site; print(site.getsitepackages())"],
         text=True,
+        encoding="utf-8",
     )
     site_packages = [str(Path(i).resolve()) for i in ast.literal_eval(out)]
 
@@ -531,6 +550,7 @@ def test_get_site_packages(tmp_path):
     out = subprocess.check_output(
         [str(session.creator.exe), "-c", r"import site; print(site.getsitepackages())"],
         text=True,
+        encoding="utf-8",
     )
     site_packages = ast.literal_eval(out)
 
@@ -569,7 +589,7 @@ def _get_sys_path(flag=None):
         if flag:
             cmd.append(flag)
         cmd.extend(["-c", "import json; import sys; print(json.dumps(sys.path))"])
-        return [i if case_sensitive else i.lower() for i in json.loads(subprocess.check_output(cmd))]
+        return [i if case_sensitive else i.lower() for i in json.loads(subprocess.check_output(cmd, encoding="utf-8"))]
 
     monkeypatch.delenv("PYTHONPATH", raising=False)
     base = _get_sys_path()

tests/unit/discovery/py_info/test_py_info.py

@@ -291,7 +291,7 @@ def test_discover_exe_on_path_non_spec_name_not_match(mocker):
     assert CURRENT.satisfies(spec, impl_must_match=True) is False
 
 
-@pytest.mark.skipif(IS_PYPY, reason="setuptools distutil1s patching does not work")
+@pytest.mark.skipif(IS_PYPY, reason="setuptools distutils patching does not work")
 def test_py_info_setuptools():
     from setuptools.dist import Distribution
 

tests/unit/discovery/windows/conftest.py

@@ -11,7 +11,7 @@ def _mock_registry(mocker):
     from virtualenv.discovery.windows.pep514 import winreg
 
     loc, glob = {}, {}
-    mock_value_str = (Path(__file__).parent / "winreg-mock-values.py").read_text()
+    mock_value_str = (Path(__file__).parent / "winreg-mock-values.py").read_text(encoding="utf-8")
     exec(mock_value_str, glob, loc)
     enum_collect = loc["enum_collect"]
     value_collect = loc["value_collect"]

tests/unit/seed/embed/test_base_embed.py

@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+import sys
+from pathlib import Path
+
 import pytest
 
 from virtualenv.run import session_via_cli
@@ -12,3 +15,13 @@
 def test_download_cli_flag(args, download, tmp_path):
     session = session_via_cli(args + [str(tmp_path)])
     assert session.seeder.download is download
+
+
+def test_embed_wheel_versions(tmp_path: Path) -> None:
+    session = session_via_cli([str(tmp_path)])
+    expected = (
+        {"pip": "bundle"}
+        if sys.version_info[:2] >= (3, 12)
+        else {"pip": "bundle", "setuptools": "bundle", "wheel": "bundle"}
+    )
+    assert session.seeder.distribution_to_versions() == expected