From f96ba08dbc8b970d9a740717c9730017eedf81f2 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 22 Jul 2022 13:22:17 +0800 Subject: [PATCH 01/10] Bump virtualenv to 20+ in tests --- .github/workflows/ci.yml | 6 +++--- tests/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7687b1b8cd4..9f5b34737a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,7 +122,7 @@ jobs: if: matrix.os == 'MacOS' run: brew install breezy - - run: pip install nox 'virtualenv<20' 'setuptools != 60.6.0' + - run: pip install nox 'virtualenv>=20' 'setuptools!=60.6.0' # Main check - name: Run unit tests @@ -179,7 +179,7 @@ jobs: $acl.AddAccessRule($rule) Set-Acl "R:\Temp" $acl - - run: pip install nox 'virtualenv<20' + - run: pip install nox 'virtualenv>=20' env: TEMP: "R:\\Temp" @@ -261,7 +261,7 @@ jobs: - name: Install Ubuntu dependencies run: sudo apt-get install bzr - - run: pip install nox 'virtualenv<20' + - run: pip install nox 'virtualenv>=20' - name: Run unit tests run: >- diff --git a/tests/requirements.txt b/tests/requirements.txt index 9ce6d62078a..f4f27e94f71 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,7 +7,7 @@ pytest-rerunfailures pytest-xdist scripttest setuptools -virtualenv < 20.0 +virtualenv >= 20.0 werkzeug wheel tomli-w From 5ded5474ac9b323496506e6391e8d8c2c888d7f1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 22 Jul 2022 13:22:39 +0800 Subject: [PATCH 02/10] Name virtualenv<20 as "legacy" Well they are. At least not "regular" anymore. --- src/pip/_internal/utils/virtualenv.py | 12 ++++++------ tests/unit/test_utils_virtualenv.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/utils/virtualenv.py b/src/pip/_internal/utils/virtualenv.py index c926db4c332..882e36f5c1d 100644 --- a/src/pip/_internal/utils/virtualenv.py +++ b/src/pip/_internal/utils/virtualenv.py @@ -19,7 +19,7 @@ def _running_under_venv() -> bool: return sys.prefix != getattr(sys, "base_prefix", sys.prefix) -def _running_under_regular_virtualenv() -> bool: +def _running_under_legacy_virtualenv() -> bool: """Checks if sys.real_prefix is set. This handles virtual environments created with pypa's virtualenv. @@ -29,8 +29,8 @@ def _running_under_regular_virtualenv() -> bool: def running_under_virtualenv() -> bool: - """Return True if we're running inside a virtualenv, False otherwise.""" - return _running_under_venv() or _running_under_regular_virtualenv() + """True if we're running inside a virtual environment, False otherwise.""" + return _running_under_venv() or _running_under_legacy_virtualenv() def _get_pyvenv_cfg_lines() -> Optional[List[str]]: @@ -77,7 +77,7 @@ def _no_global_under_venv() -> bool: return False -def _no_global_under_regular_virtualenv() -> bool: +def _no_global_under_legacy_virtualenv() -> bool: """Check if "no-global-site-packages.txt" exists beside site.py This mirrors logic in pypa/virtualenv for determining whether system @@ -98,7 +98,7 @@ def virtualenv_no_global() -> bool: if _running_under_venv(): return _no_global_under_venv() - if _running_under_regular_virtualenv(): - return _no_global_under_regular_virtualenv() + if _running_under_legacy_virtualenv(): + return _no_global_under_legacy_virtualenv() return False diff --git a/tests/unit/test_utils_virtualenv.py b/tests/unit/test_utils_virtualenv.py index 38d5383ce04..94461c6d89e 100644 --- a/tests/unit/test_utils_virtualenv.py +++ b/tests/unit/test_utils_virtualenv.py @@ -63,7 +63,7 @@ def test_virtualenv_no_global_with_regular_virtualenv( monkeypatch.setattr(site, "__file__", os.fspath(tmpdir / "site.py")) monkeypatch.setattr( virtualenv, - "_running_under_regular_virtualenv", + "_running_under_legacy_virtualenv", lambda: under_virtualenv, ) if no_global_file: @@ -73,7 +73,7 @@ def test_virtualenv_no_global_with_regular_virtualenv( @pytest.mark.parametrize( - "pyvenv_cfg_lines, under_venv, expected, expect_warning", + "pyvenv_cfg_lines, under_venv, expect_no_global, expect_warning", [ (None, False, False, False), (None, True, True, True), # this has a warning. @@ -104,15 +104,15 @@ def test_virtualenv_no_global_with_pep_405_virtual_environment( caplog: pytest.LogCaptureFixture, pyvenv_cfg_lines: Optional[List[str]], under_venv: bool, - expected: bool, + expect_no_global: bool, expect_warning: bool, ) -> None: - monkeypatch.setattr(virtualenv, "_running_under_regular_virtualenv", lambda: False) + monkeypatch.setattr(virtualenv, "_running_under_legacy_virtualenv", lambda: False) monkeypatch.setattr(virtualenv, "_get_pyvenv_cfg_lines", lambda: pyvenv_cfg_lines) monkeypatch.setattr(virtualenv, "_running_under_venv", lambda: under_venv) with caplog.at_level(logging.WARNING): - assert virtualenv.virtualenv_no_global() == expected + assert virtualenv.virtualenv_no_global() == expect_no_global if expect_warning: assert caplog.records From 1d05ba8ffdd21e24de2a01487a1318712cf66953 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 22 Jul 2022 14:11:35 +0800 Subject: [PATCH 03/10] Rewrite virtualenv tool in tests for 20+ support Co-Authored-By: Lumir Balhar --- tests/lib/venv.py | 141 +++++++++++++++------------------------------- 1 file changed, 46 insertions(+), 95 deletions(-) diff --git a/tests/lib/venv.py b/tests/lib/venv.py index ab6644bc9ab..eb54d834d2a 100644 --- a/tests/lib/venv.py +++ b/tests/lib/venv.py @@ -1,8 +1,7 @@ import compileall import os import shutil -import subprocess -import sys +import sysconfig import textwrap import venv as _venv from pathlib import Path @@ -47,15 +46,16 @@ def __init__( self._create() def _update_paths(self) -> None: - home, lib, inc, bin = _virtualenv.path_locations(self.location) - self.bin = Path(bin) - self.site = Path(lib) / "site-packages" - # Workaround for https://github.com/pypa/virtualenv/issues/306 - if hasattr(sys, "pypy_version_info"): - version_dir = str(sys.version_info.major) - self.lib = Path(home, "lib-python", version_dir) - else: - self.lib = Path(lib) + bases = { + "installed_base": self.location, + "installed_platbase": self.location, + "base": self.location, + "platbase": self.location, + } + paths = sysconfig.get_paths(vars=bases) + self.bin = Path(paths["scripts"]) + self.site = Path(paths["purelib"]) + self.lib = Path(paths["stdlib"]) def __repr__(self) -> str: return f"" @@ -64,10 +64,6 @@ def _create(self, clear: bool = False) -> None: if clear: shutil.rmtree(self.location) if self._template: - # On Windows, calling `_virtualenv.path_locations(target)` - # will have created the `target` directory... - if sys.platform == "win32" and self.location.exists(): - self.location.rmdir() # Clone virtual environment from template. shutil.copytree(self._template.location, self.location, symlinks=True) self._sitecustomize = self._template.sitecustomize @@ -75,18 +71,14 @@ def _create(self, clear: bool = False) -> None: else: # Create a new virtual environment. if self._venv_type == "virtualenv": - subprocess.check_call( + _virtualenv.cli_run( [ - sys.executable, - "-m", - "virtualenv", "--no-pip", "--no-wheel", "--no-setuptools", - str(self.location), - ] + os.fspath(self.location), + ], ) - self._fix_virtualenv_site_module() elif self._venv_type == "venv": builder = _venv.EnvBuilder() context = builder.ensure_directories(self.location) @@ -96,71 +88,30 @@ def _create(self, clear: bool = False) -> None: self.sitecustomize = self._sitecustomize self.user_site_packages = self._user_site_packages - def _fix_virtualenv_site_module(self) -> None: - # Patch `site.py` so user site work as expected. - site_py = self.lib / "site.py" - with open(site_py) as fp: - site_contents = fp.read() - for pattern, replace in ( - ( - # Ensure enabling user site does not result in adding - # the real site-packages' directory to `sys.path`. - ("\ndef virtual_addsitepackages(known_paths):\n"), - ( - "\ndef virtual_addsitepackages(known_paths):\n" - " return known_paths\n" - ), - ), - ( - # Fix sites ordering: user site must be added before system. - ( - "\n paths_in_sys = addsitepackages(paths_in_sys)" - "\n paths_in_sys = addusersitepackages(paths_in_sys)\n" - ), - ( - "\n paths_in_sys = addusersitepackages(paths_in_sys)" - "\n paths_in_sys = addsitepackages(paths_in_sys)\n" - ), - ), - ): - assert pattern in site_contents - site_contents = site_contents.replace(pattern, replace) - with open(site_py, "w") as fp: - fp.write(site_contents) - # Make sure bytecode is up-to-date too. - assert compileall.compile_file(str(site_py), quiet=1, force=True) - def _customize_site(self) -> None: - contents = "" - if self._venv_type == "venv": - # Enable user site (before system). - contents += textwrap.dedent( - """ - import os, site, sys - - if not os.environ.get('PYTHONNOUSERSITE', False): - - site.ENABLE_USER_SITE = True - - # First, drop system-sites related paths. - original_sys_path = sys.path[:] - known_paths = set() - for path in site.getsitepackages(): - site.addsitedir(path, known_paths=known_paths) - system_paths = sys.path[len(original_sys_path):] - for path in system_paths: - if path in original_sys_path: - original_sys_path.remove(path) - sys.path = original_sys_path - - # Second, add user-site. - site.addsitedir(site.getusersitepackages()) - - # Third, add back system-sites related paths. - for path in site.getsitepackages(): - site.addsitedir(path) - """ - ).strip() + # Enable user site (before system). + contents = textwrap.dedent( + """ + import os, site, sys + if not os.environ.get('PYTHONNOUSERSITE', False): + site.ENABLE_USER_SITE = True + # First, drop system-sites related paths. + original_sys_path = sys.path[:] + known_paths = set() + for path in site.getsitepackages(): + site.addsitedir(path, known_paths=known_paths) + system_paths = sys.path[len(original_sys_path):] + for path in system_paths: + if path in original_sys_path: + original_sys_path.remove(path) + sys.path = original_sys_path + # Second, add user-site. + site.addsitedir(site.getusersitepackages()) + # Third, add back system-sites related paths. + for path in site.getsitepackages(): + site.addsitedir(path) + """ + ).strip() if self._sitecustomize is not None: contents += "\n" + self._sitecustomize sitecustomize = self.site / "sitecustomize.py" @@ -191,12 +142,12 @@ def user_site_packages(self) -> bool: @user_site_packages.setter def user_site_packages(self, value: bool) -> None: - self._user_site_packages = value - if self._venv_type == "virtualenv": - marker = self.lib / "no-global-site-packages.txt" - if self._user_site_packages: - marker.unlink() - else: - marker.touch() - elif self._venv_type == "venv": - self._customize_site() + self._customize_site() + pyvenv_cfg = self.location.joinpath("pyvenv.cfg") + modified_lines = [] + for line in pyvenv_cfg.read_text().splitlines(): + k, v = line.split("=", 1) + if k.strip() == "include-system-site-packages": + line = f"include-system-site-packages = {str(bool(value)).lower()}" + modified_lines.append(line) + pyvenv_cfg.write_text("\n".join(modified_lines)) From 4d533cc269ae501ba4e29735f0d5f69b533ecafb Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 22 Jul 2022 18:04:28 +0800 Subject: [PATCH 04/10] Don't enable global site when enabling user site --- tests/lib/venv.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/lib/venv.py b/tests/lib/venv.py index eb54d834d2a..9177651d125 100644 --- a/tests/lib/venv.py +++ b/tests/lib/venv.py @@ -91,10 +91,10 @@ def _create(self, clear: bool = False) -> None: def _customize_site(self) -> None: # Enable user site (before system). contents = textwrap.dedent( - """ + f""" import os, site, sys if not os.environ.get('PYTHONNOUSERSITE', False): - site.ENABLE_USER_SITE = True + site.ENABLE_USER_SITE = {self._user_site_packages} # First, drop system-sites related paths. original_sys_path = sys.path[:] known_paths = set() @@ -106,7 +106,8 @@ def _customize_site(self) -> None: original_sys_path.remove(path) sys.path = original_sys_path # Second, add user-site. - site.addsitedir(site.getusersitepackages()) + if {self._user_site_packages}: + site.addsitedir(site.getusersitepackages()) # Third, add back system-sites related paths. for path in site.getsitepackages(): site.addsitedir(path) @@ -142,12 +143,5 @@ def user_site_packages(self) -> bool: @user_site_packages.setter def user_site_packages(self, value: bool) -> None: + self._user_site_packages = value self._customize_site() - pyvenv_cfg = self.location.joinpath("pyvenv.cfg") - modified_lines = [] - for line in pyvenv_cfg.read_text().splitlines(): - k, v = line.split("=", 1) - if k.strip() == "include-system-site-packages": - line = f"include-system-site-packages = {str(bool(value)).lower()}" - modified_lines.append(line) - pyvenv_cfg.write_text("\n".join(modified_lines)) From 4f20a7e3e76674c3361d8bde175fff82ed301d35 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 27 Oct 2022 09:15:33 +0800 Subject: [PATCH 05/10] Set include-system-site-packages for user site --- tests/lib/venv.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/lib/venv.py b/tests/lib/venv.py index 9177651d125..55b81a7d4b5 100644 --- a/tests/lib/venv.py +++ b/tests/lib/venv.py @@ -5,7 +5,7 @@ import textwrap import venv as _venv from pathlib import Path -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING, Dict, Optional, Union import virtualenv as _virtualenv @@ -120,6 +120,21 @@ def _customize_site(self) -> None: # Make sure bytecode is up-to-date too. assert compileall.compile_file(str(sitecustomize), quiet=1, force=True) + def _rewrite_pyvenv_cfg(self, replacements: Dict[str, str]) -> None: + pyvenv_cfg = self.location.joinpath("pyvenv.cfg") + lines = pyvenv_cfg.read_text(encoding="utf-8").splitlines() + + def maybe_replace_line(line: str) -> str: + key = line.split("=", 1)[0].strip() + try: + value = replacements[key] + except KeyError: # No need to replace. + return line + return f"{key} = {value}" + + lines = [maybe_replace_line(line) for line in lines] + pyvenv_cfg.write_text("\n".join(lines), encoding="utf-8") + def clear(self) -> None: self._create(clear=True) @@ -144,4 +159,7 @@ def user_site_packages(self) -> bool: @user_site_packages.setter def user_site_packages(self, value: bool) -> None: self._user_site_packages = value + self._rewrite_pyvenv_cfg( + {"include-system-site-packages": str(bool(value)).lower()} + ) self._customize_site() From b850d539f9890f5283fffce1a625149ae2a1e5c3 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 27 Oct 2022 09:29:50 +0800 Subject: [PATCH 06/10] Keep using old virtualenv for Python < 3.10 pip uses distutils (instead of sysconfig) for Python < 3.10, which has awkward path issues when faking a user site. --- tests/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index f4f27e94f71..84b7c14d4b4 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,7 +7,8 @@ pytest-rerunfailures pytest-xdist scripttest setuptools -virtualenv >= 20.0 +virtualenv < 20.0 ; python_version < '3.10' +virtualenv >= 20.0 ; python_version >= '3.10' werkzeug wheel tomli-w From 4bf1b67964aad6f5db2ec5888833c30685b6d4dd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 27 Oct 2022 10:02:23 +0800 Subject: [PATCH 07/10] Build fake wheels for --user tests The old INITools tests rely on setup.py, which relies on distutils and generates a ton of issues. Build fake wheels directly to avoid dealing with them. --- tests/functional/test_install_user.py | 135 +++++++++++++++++--------- 1 file changed, 91 insertions(+), 44 deletions(-) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index d0bdbc3a547..e34b35431dc 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -1,6 +1,7 @@ """ tests specific to "pip install --user" """ +import os import textwrap from os.path import curdir, isdir, isfile from pathlib import Path @@ -8,7 +9,12 @@ import pytest from tests.lib import pyversion # noqa: F401 -from tests.lib import PipTestEnvironment, TestData, need_svn +from tests.lib import ( + PipTestEnvironment, + TestData, + create_basic_wheel_for_package, + need_svn, +) from tests.lib.local_repos import local_checkout from tests.lib.venv import VirtualEnvironment @@ -142,7 +148,6 @@ def test_install_user_conflict_in_usersite( result2.did_create(egg_info_folder) assert not isfile(initools_v3_file), initools_v3_file - @pytest.mark.network @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_globalsite( self, virtualenv: VirtualEnvironment, script: PipTestEnvironment @@ -151,30 +156,41 @@ def test_install_user_conflict_in_globalsite( Test user install with conflict in global site ignores site and installs to usersite """ - _patch_dist_in_site_packages(virtualenv) + create_basic_wheel_for_package(script, "initools", "0.1") + create_basic_wheel_for_package(script, "initools", "0.2") - script.pip("install", "INITools==0.2", "--no-binary=:all:") + _patch_dist_in_site_packages(virtualenv) - result2 = script.pip("install", "--user", "INITools==0.1", "--no-binary=:all:") + script.pip( + "install", + "--no-index", + "--find-links", + script.scratch_path, + "initools==0.2", + ) + result2 = script.pip( + "install", + "--no-index", + "--find-links", + script.scratch_path, + "--user", + "initools==0.1", + ) # usersite has 0.1 - # we still test for egg-info because no-binary implies setup.py install - egg_info_folder = script.user_site / f"INITools-0.1-py{pyversion}.egg-info" + dist_info_folder = script.user_site / "initools-0.1.dist-info" initools_folder = script.user_site / "initools" - result2.did_create(egg_info_folder) + result2.did_create(dist_info_folder) result2.did_create(initools_folder) # site still has 0.2 (can't look in result1; have to check) - egg_info_folder = ( - script.base_path - / script.site_packages - / f"INITools-0.2-py{pyversion}.egg-info" + dist_info_folder = ( + script.base_path / script.site_packages / "initools-0.2.dist-info" ) initools_folder = script.base_path / script.site_packages / "initools" - assert isdir(egg_info_folder) + assert isdir(dist_info_folder) assert isdir(initools_folder) - @pytest.mark.network @pytest.mark.incompatible_with_test_venv def test_upgrade_user_conflict_in_globalsite( self, virtualenv: VirtualEnvironment, script: PipTestEnvironment @@ -183,31 +199,42 @@ def test_upgrade_user_conflict_in_globalsite( Test user install/upgrade with conflict in global site ignores site and installs to usersite """ + create_basic_wheel_for_package(script, "initools", "0.2") + create_basic_wheel_for_package(script, "initools", "0.3.1") + _patch_dist_in_site_packages(virtualenv) - script.pip("install", "INITools==0.2", "--no-binary=:all:") + script.pip( + "install", + "--no-index", + "--find-links", + script.scratch_path, + "initools==0.2", + ) result2 = script.pip( - "install", "--user", "--upgrade", "INITools", "--no-binary=:all:" + "install", + "--no-index", + "--find-links", + script.scratch_path, + "--user", + "--upgrade", + "initools", ) # usersite has 0.3.1 - # we still test for egg-info because no-binary implies setup.py install - egg_info_folder = script.user_site / f"INITools-0.3.1-py{pyversion}.egg-info" + dist_info_folder = script.user_site / "initools-0.3.1.dist-info" initools_folder = script.user_site / "initools" - result2.did_create(egg_info_folder) + result2.did_create(dist_info_folder) result2.did_create(initools_folder) # site still has 0.2 (can't look in result1; have to check) - egg_info_folder = ( - script.base_path - / script.site_packages - / f"INITools-0.2-py{pyversion}.egg-info" + dist_info_folder = ( + script.base_path / script.site_packages / "initools-0.2.dist-info" ) initools_folder = script.base_path / script.site_packages / "initools" - assert isdir(egg_info_folder), result2.stdout + assert isdir(dist_info_folder), result2.stdout assert isdir(initools_folder) - @pytest.mark.network @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_globalsite_and_usersite( self, virtualenv: VirtualEnvironment, script: PipTestEnvironment @@ -216,34 +243,54 @@ def test_install_user_conflict_in_globalsite_and_usersite( Test user install with conflict in globalsite and usersite ignores global site and updates usersite. """ - _patch_dist_in_site_packages(virtualenv) + initools_v3_file_name = os.path.join("initools", "configparser.py") + create_basic_wheel_for_package(script, "initools", "0.1") + create_basic_wheel_for_package(script, "initools", "0.2") + create_basic_wheel_for_package( + script, + "initools", + "0.3", + extra_files={initools_v3_file_name: "# Hi!"}, + ) - script.pip("install", "INITools==0.2", "--no-binary=:all:") - script.pip("install", "--user", "INITools==0.3", "--no-binary=:all:") + _patch_dist_in_site_packages(virtualenv) - result3 = script.pip("install", "--user", "INITools==0.1", "--no-binary=:all:") + script.pip( + "install", + "--no-index", + "--find-links", + script.scratch_path, + "initools==0.2", + ) + script.pip( + "install", + "--no-index", + "--find-links", + script.scratch_path, + "--user", + "initools==0.3", + ) + result3 = script.pip( + "install", + "--no-index", + "--find-links", + script.scratch_path, + "--user", + "initools==0.1", + ) # usersite has 0.1 - # we still test for egg-info because no-binary implies setup.py install - egg_info_folder = script.user_site / f"INITools-0.1-py{pyversion}.egg-info" - initools_v3_file = ( - # file only in 0.3 - script.base_path - / script.user_site - / "initools" - / "configparser.py" - ) - result3.did_create(egg_info_folder) + dist_info_folder = script.user_site / "initools-0.1.dist-info" + result3.did_create(dist_info_folder) + initools_v3_file = script.base_path / script.user_site / initools_v3_file_name assert not isfile(initools_v3_file), initools_v3_file # site still has 0.2 (can't just look in result1; have to check) - egg_info_folder = ( - script.base_path - / script.site_packages - / f"INITools-0.2-py{pyversion}.egg-info" + dist_info_folder = ( + script.base_path / script.site_packages / "initools-0.2.dist-info" ) initools_folder = script.base_path / script.site_packages / "initools" - assert isdir(egg_info_folder) + assert isdir(dist_info_folder) assert isdir(initools_folder) @pytest.mark.network From 4ab48650633b61e2c4f730584553604758724fb2 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 27 Oct 2022 10:36:46 +0800 Subject: [PATCH 08/10] One more rewrite to avoid distutils --- tests/functional/test_uninstall_user.py | 44 ++++++++++++++++++------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/tests/functional/test_uninstall_user.py b/tests/functional/test_uninstall_user.py index 6d48fe1627a..1ef65dd1671 100644 --- a/tests/functional/test_uninstall_user.py +++ b/tests/functional/test_uninstall_user.py @@ -6,9 +6,9 @@ import pytest from tests.functional.test_install_user import _patch_dist_in_site_packages -from tests.lib import pyversion # noqa: F401 from tests.lib import PipTestEnvironment, TestData, assert_all_changes from tests.lib.venv import VirtualEnvironment +from tests.lib.wheel import make_wheel @pytest.mark.incompatible_with_test_venv @@ -28,14 +28,39 @@ def test_uninstall_from_usersite_with_dist_in_global_site( """ Test uninstall from usersite (with same dist in global site) """ + entry_points_txt = "[console_scripts]\nscript = pkg:func" + make_wheel( + "pkg", + "0.1", + extra_metadata_files={"entry_points.txt": entry_points_txt}, + ).save_to_dir(script.scratch_path) + make_wheel( + "pkg", + "0.1.1", + extra_metadata_files={"entry_points.txt": entry_points_txt}, + ).save_to_dir(script.scratch_path) + _patch_dist_in_site_packages(virtualenv) - script.pip_install_local("pip-test-package==0.1", "--no-binary=:all:") + script.pip( + "install", + "--no-index", + "--find-links", + script.scratch_path, + "--no-warn-script-location", + "pkg==0.1", + ) - result2 = script.pip_install_local( - "--user", "pip-test-package==0.1.1", "--no-binary=:all:" + result2 = script.pip( + "install", + "--no-index", + "--find-links", + script.scratch_path, + "--no-warn-script-location", + "--user", + "pkg==0.1.1", ) - result3 = script.pip("uninstall", "-vy", "pip-test-package") + result3 = script.pip("uninstall", "-vy", "pkg") # uninstall console is mentioning user scripts, but not global scripts assert normcase(script.user_bin_path) in result3.stdout, str(result3) @@ -45,13 +70,8 @@ def test_uninstall_from_usersite_with_dist_in_global_site( assert_all_changes(result2, result3, [script.venv / "build", "cache"]) # site still has 0.2 (can't look in result1; have to check) - # keep checking for egg-info because no-binary implies setup.py install - egg_info_folder = ( - script.base_path - / script.site_packages - / f"pip_test_package-0.1-py{pyversion}.egg-info" - ) - assert isdir(egg_info_folder) + dist_info_folder = script.base_path / script.site_packages / "pkg-0.1.dist-info" + assert isdir(dist_info_folder) def test_uninstall_editable_from_usersite( self, script: PipTestEnvironment, data: TestData From 83c85e94b70aa5b3211ef3cc592d8cd8619beb15 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 27 Oct 2022 11:02:51 +0800 Subject: [PATCH 09/10] Fix legacy virtualenv setup in tests --- .github/workflows/ci.yml | 6 +- tests/lib/venv.py | 154 ++++++++++++++++++++++++++++++--------- 2 files changed, 121 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f5b34737a7..c9cfb7b8e98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,7 +122,7 @@ jobs: if: matrix.os == 'MacOS' run: brew install breezy - - run: pip install nox 'virtualenv>=20' 'setuptools!=60.6.0' + - run: pip install nox # Main check - name: Run unit tests @@ -179,7 +179,7 @@ jobs: $acl.AddAccessRule($rule) Set-Acl "R:\Temp" $acl - - run: pip install nox 'virtualenv>=20' + - run: pip install nox env: TEMP: "R:\\Temp" @@ -261,7 +261,7 @@ jobs: - name: Install Ubuntu dependencies run: sudo apt-get install bzr - - run: pip install nox 'virtualenv>=20' + - run: pip install nox - name: Run unit tests run: >- diff --git a/tests/lib/venv.py b/tests/lib/venv.py index 55b81a7d4b5..42ec5082ad7 100644 --- a/tests/lib/venv.py +++ b/tests/lib/venv.py @@ -1,6 +1,8 @@ import compileall import os import shutil +import subprocess +import sys import sysconfig import textwrap import venv as _venv @@ -18,6 +20,9 @@ VirtualEnvironmentType = str +LEGACY_VIRTUALENV = int(_virtualenv.__version__.split(".", 1)[0]) < 20 + + class VirtualEnvironment: """ An abstraction around virtual environments, currently it only uses @@ -39,13 +44,28 @@ def __init__( self._venv_type = venv_type else: self._venv_type = "virtualenv" + assert self._venv_type in ("virtualenv", "venv") self._user_site_packages = False self._template = template self._sitecustomize: Optional[str] = None self._update_paths() self._create() + def __update_paths_legacy(self) -> None: + home, lib, inc, bin = _virtualenv.path_locations(self.location) + self.bin = Path(bin) + self.site = Path(lib) / "site-packages" + # Workaround for https://github.com/pypa/virtualenv/issues/306 + if hasattr(sys, "pypy_version_info"): + version_dir = str(sys.version_info.major) + self.lib = Path(home, "lib-python", version_dir) + else: + self.lib = Path(lib) + def _update_paths(self) -> None: + if LEGACY_VIRTUALENV: + self.__update_paths_legacy() + return bases = { "installed_base": self.location, "installed_platbase": self.location, @@ -64,6 +84,10 @@ def _create(self, clear: bool = False) -> None: if clear: shutil.rmtree(self.location) if self._template: + # On Windows, calling `_virtualenv.path_locations(target)` + # will have created the `target` directory... + if LEGACY_VIRTUALENV and sys.platform == "win32" and self.location.exists(): + self.location.rmdir() # Clone virtual environment from template. shutil.copytree(self._template.location, self.location, symlinks=True) self._sitecustomize = self._template.sitecustomize @@ -71,14 +95,28 @@ def _create(self, clear: bool = False) -> None: else: # Create a new virtual environment. if self._venv_type == "virtualenv": - _virtualenv.cli_run( - [ - "--no-pip", - "--no-wheel", - "--no-setuptools", - os.fspath(self.location), - ], - ) + if LEGACY_VIRTUALENV: + subprocess.check_call( + [ + sys.executable, + "-m", + "virtualenv", + "--no-pip", + "--no-wheel", + "--no-setuptools", + os.fspath(self.location), + ] + ) + self._fix_legacy_virtualenv_site_module() + else: + _virtualenv.cli_run( + [ + "--no-pip", + "--no-wheel", + "--no-setuptools", + os.fspath(self.location), + ], + ) elif self._venv_type == "venv": builder = _venv.EnvBuilder() context = builder.ensure_directories(self.location) @@ -88,31 +126,68 @@ def _create(self, clear: bool = False) -> None: self.sitecustomize = self._sitecustomize self.user_site_packages = self._user_site_packages + def _fix_legacy_virtualenv_site_module(self) -> None: + # Patch `site.py` so user site work as expected. + site_py = self.lib / "site.py" + with open(site_py) as fp: + site_contents = fp.read() + for pattern, replace in ( + ( + # Ensure enabling user site does not result in adding + # the real site-packages' directory to `sys.path`. + ("\ndef virtual_addsitepackages(known_paths):\n"), + ( + "\ndef virtual_addsitepackages(known_paths):\n" + " return known_paths\n" + ), + ), + ( + # Fix sites ordering: user site must be added before system. + ( + "\n paths_in_sys = addsitepackages(paths_in_sys)" + "\n paths_in_sys = addusersitepackages(paths_in_sys)\n" + ), + ( + "\n paths_in_sys = addusersitepackages(paths_in_sys)" + "\n paths_in_sys = addsitepackages(paths_in_sys)\n" + ), + ), + ): + assert pattern in site_contents + site_contents = site_contents.replace(pattern, replace) + with open(site_py, "w") as fp: + fp.write(site_contents) + # Make sure bytecode is up-to-date too. + assert compileall.compile_file(str(site_py), quiet=1, force=True) + def _customize_site(self) -> None: - # Enable user site (before system). - contents = textwrap.dedent( - f""" - import os, site, sys - if not os.environ.get('PYTHONNOUSERSITE', False): - site.ENABLE_USER_SITE = {self._user_site_packages} - # First, drop system-sites related paths. - original_sys_path = sys.path[:] - known_paths = set() - for path in site.getsitepackages(): - site.addsitedir(path, known_paths=known_paths) - system_paths = sys.path[len(original_sys_path):] - for path in system_paths: - if path in original_sys_path: - original_sys_path.remove(path) - sys.path = original_sys_path - # Second, add user-site. - if {self._user_site_packages}: - site.addsitedir(site.getusersitepackages()) - # Third, add back system-sites related paths. - for path in site.getsitepackages(): - site.addsitedir(path) - """ - ).strip() + if not LEGACY_VIRTUALENV or self._venv_type == "venv": + # Enable user site (before system). + contents = textwrap.dedent( + f""" + import os, site, sys + if not os.environ.get('PYTHONNOUSERSITE', False): + site.ENABLE_USER_SITE = {self._user_site_packages} + # First, drop system-sites related paths. + original_sys_path = sys.path[:] + known_paths = set() + for path in site.getsitepackages(): + site.addsitedir(path, known_paths=known_paths) + system_paths = sys.path[len(original_sys_path):] + for path in system_paths: + if path in original_sys_path: + original_sys_path.remove(path) + sys.path = original_sys_path + # Second, add user-site. + if {self._user_site_packages}: + site.addsitedir(site.getusersitepackages()) + # Third, add back system-sites related paths. + for path in site.getsitepackages(): + site.addsitedir(path) + """ + ).strip() + else: + contents = "" if self._sitecustomize is not None: contents += "\n" + self._sitecustomize sitecustomize = self.site / "sitecustomize.py" @@ -159,7 +234,14 @@ def user_site_packages(self) -> bool: @user_site_packages.setter def user_site_packages(self, value: bool) -> None: self._user_site_packages = value - self._rewrite_pyvenv_cfg( - {"include-system-site-packages": str(bool(value)).lower()} - ) - self._customize_site() + if not LEGACY_VIRTUALENV or self._venv_type == "venv": + self._rewrite_pyvenv_cfg( + {"include-system-site-packages": str(bool(value)).lower()} + ) + self._customize_site() + else: + marker = self.lib / "no-global-site-packages.txt" + if self._user_site_packages: + marker.unlink() + else: + marker.touch() From 50e194f1070733d0af66904001a89e4d603387b4 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 27 Oct 2022 21:34:17 +0800 Subject: [PATCH 10/10] Selectively enable user site The modern virtual environment structure does not allow us to enable "fake user site" while disabling the global site, so we need to do more fine-grained configuration to correctly set up test environments for each test case. With this done, we can also properly support the stdlib venv ad the test environment backend, since it basically works identically with modern virtualenv. The incompatible_with_test_venv is thus removed. --- setup.cfg | 1 - tests/conftest.py | 12 ++- tests/functional/test_build_env.py | 2 +- tests/functional/test_freeze.py | 4 +- tests/functional/test_install.py | 4 +- tests/functional/test_install_reqs.py | 3 +- tests/functional/test_install_user.py | 9 +-- tests/functional/test_install_wheel.py | 3 +- tests/functional/test_list.py | 6 +- tests/functional/test_new_resolver_user.py | 17 ++--- tests/functional/test_uninstall_user.py | 2 +- tests/lib/venv.py | 85 ++++++++++++---------- 12 files changed, 70 insertions(+), 78 deletions(-) diff --git a/setup.cfg b/setup.cfg index dae2f21b10d..1502abfc86a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,7 +63,6 @@ xfail_strict = True markers = network: tests that need network incompatible_with_sysconfig - incompatible_with_test_venv incompatible_with_venv no_auto_tempdir_manager unit: unit tests diff --git a/tests/conftest.py b/tests/conftest.py index 44aa56026b6..46975b29beb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -108,10 +108,6 @@ def pytest_collection_modifyitems(config: Config, items: List[pytest.Function]) if item.get_closest_marker("network") is not None: item.add_marker(pytest.mark.flaky(reruns=3, reruns_delay=2)) - if item.get_closest_marker("incompatible_with_test_venv") and config.getoption( - "--use-venv" - ): - item.add_marker(pytest.mark.skip("Incompatible with test venv")) if ( item.get_closest_marker("incompatible_with_venv") and sys.prefix != sys.base_prefix @@ -474,9 +470,6 @@ def virtualenv_template( ): (venv.bin / exe).unlink() - # Enable user site packages. - venv.user_site_packages = True - # Rename original virtualenv directory to make sure # it's not reused by mistake from one of the copies. venv_template = tmpdir / "venv_template" @@ -742,3 +735,8 @@ def mock_server() -> Iterator[MockServer]: @pytest.fixture def proxy(request: pytest.FixtureRequest) -> str: return request.config.getoption("proxy") + + +@pytest.fixture +def enable_user_site(virtualenv: VirtualEnvironment) -> None: + virtualenv.user_site_packages = True diff --git a/tests/functional/test_build_env.py b/tests/functional/test_build_env.py index 869e8ad921d..93a6b930f66 100644 --- a/tests/functional/test_build_env.py +++ b/tests/functional/test_build_env.py @@ -204,7 +204,7 @@ def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> No assert result.stdout.strip() == "2.0", str(result) -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_build_env_isolation(script: PipTestEnvironment) -> None: # Create dummy `pkg` wheel. diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 535581121ff..49b362d7e96 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -862,7 +862,7 @@ def test_freeze_with_requirement_option_package_repeated_multi_file( @pytest.mark.network -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_freeze_user( script: PipTestEnvironment, virtualenv: VirtualEnvironment, data: TestData ) -> None: @@ -900,7 +900,7 @@ def test_freeze_path(tmpdir: Path, script: PipTestEnvironment, data: TestData) - @pytest.mark.network -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_freeze_path_exclude_user( tmpdir: Path, script: PipTestEnvironment, data: TestData ) -> None: diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 876f2e12a7c..f611372685f 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -171,7 +171,7 @@ def test_pep518_allows_missing_requires( assert result.files_created -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_pep518_with_user_pip( script: PipTestEnvironment, pip_src: Path, data: TestData, common_wheels: Path ) -> None: @@ -2106,7 +2106,7 @@ def test_target_install_ignores_distutils_config_install_prefix( result.did_not_create(relative_script_base) -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_user_config_accepted(script: PipTestEnvironment) -> None: # user set in the config file is parsed as 0/1 instead of True/False. # Check that this doesn't cause a problem. diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index 19c526aab09..14e1056ae7a 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -305,8 +305,7 @@ def test_install_local_with_subdirectory(script: PipTestEnvironment) -> None: result.assert_installed("version_subpkg.py", editable=False) -@pytest.mark.incompatible_with_test_venv -@pytest.mark.usefixtures("with_wheel") +@pytest.mark.usefixtures("enable_user_site", "with_wheel") def test_wheel_user_with_prefix_in_pydistutils_cfg( script: PipTestEnvironment, data: TestData ) -> None: diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index e34b35431dc..c960d0de4f9 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -35,9 +35,9 @@ def dist_in_site_packages(dist): ) +@pytest.mark.usefixtures("enable_user_site") class Tests_UserSite: @pytest.mark.network - @pytest.mark.incompatible_with_test_venv def test_reset_env_system_site_packages_usersite( self, script: PipTestEnvironment ) -> None: @@ -57,7 +57,6 @@ def test_reset_env_system_site_packages_usersite( @pytest.mark.xfail @pytest.mark.network @need_svn - @pytest.mark.incompatible_with_test_venv def test_install_subversion_usersite_editable_with_distribute( self, script: PipTestEnvironment, tmpdir: Path ) -> None: @@ -77,7 +76,6 @@ def test_install_subversion_usersite_editable_with_distribute( ) result.assert_installed("INITools", use_user_site=True) - @pytest.mark.incompatible_with_test_venv @pytest.mark.usefixtures("with_wheel") def test_install_from_current_directory_into_usersite( self, script: PipTestEnvironment, data: TestData @@ -123,7 +121,6 @@ def test_install_user_venv_nositepkgs_fails( ) @pytest.mark.network - @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_usersite( self, script: PipTestEnvironment ) -> None: @@ -148,7 +145,6 @@ def test_install_user_conflict_in_usersite( result2.did_create(egg_info_folder) assert not isfile(initools_v3_file), initools_v3_file - @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_globalsite( self, virtualenv: VirtualEnvironment, script: PipTestEnvironment ) -> None: @@ -191,7 +187,6 @@ def test_install_user_conflict_in_globalsite( assert isdir(dist_info_folder) assert isdir(initools_folder) - @pytest.mark.incompatible_with_test_venv def test_upgrade_user_conflict_in_globalsite( self, virtualenv: VirtualEnvironment, script: PipTestEnvironment ) -> None: @@ -235,7 +230,6 @@ def test_upgrade_user_conflict_in_globalsite( assert isdir(dist_info_folder), result2.stdout assert isdir(initools_folder) - @pytest.mark.incompatible_with_test_venv def test_install_user_conflict_in_globalsite_and_usersite( self, virtualenv: VirtualEnvironment, script: PipTestEnvironment ) -> None: @@ -294,7 +288,6 @@ def test_install_user_conflict_in_globalsite_and_usersite( assert isdir(initools_folder) @pytest.mark.network - @pytest.mark.incompatible_with_test_venv def test_install_user_in_global_virtualenv_with_conflict_fails( self, script: PipTestEnvironment ) -> None: diff --git a/tests/functional/test_install_wheel.py b/tests/functional/test_install_wheel.py index e988e741962..49c2d1d6d7c 100644 --- a/tests/functional/test_install_wheel.py +++ b/tests/functional/test_install_wheel.py @@ -406,8 +406,7 @@ def test_wheel_record_lines_have_updated_hash_for_scripts( ] -@pytest.mark.incompatible_with_test_venv -@pytest.mark.usefixtures("with_wheel") +@pytest.mark.usefixtures("enable_user_site", "with_wheel") def test_install_user_wheel( script: PipTestEnvironment, shared_data: TestData, tmpdir: Path ) -> None: diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index c7fdec2f2fc..d05fe9dcea5 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -129,7 +129,7 @@ def test_multiple_exclude_and_normalization( @pytest.mark.network -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_user_flag(script: PipTestEnvironment, data: TestData) -> None: """ Test the behavior of --user flag in the list command @@ -144,7 +144,7 @@ def test_user_flag(script: PipTestEnvironment, data: TestData) -> None: @pytest.mark.network -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_user_columns_flag(script: PipTestEnvironment, data: TestData) -> None: """ Test the behavior of --user --format=columns flags in the list command @@ -656,7 +656,7 @@ def test_list_path(tmpdir: Path, script: PipTestEnvironment, data: TestData) -> assert {"name": "simple", "version": "2.0"} in json_result -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_list_path_exclude_user( tmpdir: Path, script: PipTestEnvironment, data: TestData ) -> None: diff --git a/tests/functional/test_new_resolver_user.py b/tests/functional/test_new_resolver_user.py index 2f9fb65ba5a..4578c311468 100644 --- a/tests/functional/test_new_resolver_user.py +++ b/tests/functional/test_new_resolver_user.py @@ -7,7 +7,7 @@ from tests.lib.venv import VirtualEnvironment -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_new_resolver_install_user(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "base", "0.1.0") result = script.pip( @@ -22,7 +22,7 @@ def test_new_resolver_install_user(script: PipTestEnvironment) -> None: result.did_create(script.user_site / "base") -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_new_resolver_install_user_satisfied_by_global_site( script: PipTestEnvironment, ) -> None: @@ -53,7 +53,7 @@ def test_new_resolver_install_user_satisfied_by_global_site( result.did_not_create(script.user_site / "base") -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_new_resolver_install_user_conflict_in_user_site( script: PipTestEnvironment, ) -> None: @@ -91,7 +91,7 @@ def test_new_resolver_install_user_conflict_in_user_site( result.did_not_create(base_2_dist_info) -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") def test_new_resolver_install_user_in_virtualenv_with_conflict_fails( script: PipTestEnvironment, ) -> None: @@ -141,8 +141,7 @@ def dist_in_site_packages(dist): ) -@pytest.mark.incompatible_with_test_venv -@pytest.mark.usefixtures("patch_dist_in_site_packages") +@pytest.mark.usefixtures("enable_user_site", "patch_dist_in_site_packages") def test_new_resolver_install_user_reinstall_global_site( script: PipTestEnvironment, ) -> None: @@ -177,8 +176,7 @@ def test_new_resolver_install_user_reinstall_global_site( assert "base" in site_packages_content -@pytest.mark.incompatible_with_test_venv -@pytest.mark.usefixtures("patch_dist_in_site_packages") +@pytest.mark.usefixtures("enable_user_site", "patch_dist_in_site_packages") def test_new_resolver_install_user_conflict_in_global_site( script: PipTestEnvironment, ) -> None: @@ -215,8 +213,7 @@ def test_new_resolver_install_user_conflict_in_global_site( assert "base-1.0.0.dist-info" in site_packages_content -@pytest.mark.incompatible_with_test_venv -@pytest.mark.usefixtures("patch_dist_in_site_packages") +@pytest.mark.usefixtures("enable_user_site", "patch_dist_in_site_packages") def test_new_resolver_install_user_conflict_in_global_and_user_sites( script: PipTestEnvironment, ) -> None: diff --git a/tests/functional/test_uninstall_user.py b/tests/functional/test_uninstall_user.py index 1ef65dd1671..0bf2e6d4180 100644 --- a/tests/functional/test_uninstall_user.py +++ b/tests/functional/test_uninstall_user.py @@ -11,7 +11,7 @@ from tests.lib.wheel import make_wheel -@pytest.mark.incompatible_with_test_venv +@pytest.mark.usefixtures("enable_user_site") class Tests_UninstallUserSite: @pytest.mark.network def test_uninstall_from_usersite(self, script: PipTestEnvironment) -> None: diff --git a/tests/lib/venv.py b/tests/lib/venv.py index 42ec5082ad7..e65a3291230 100644 --- a/tests/lib/venv.py +++ b/tests/lib/venv.py @@ -20,9 +20,6 @@ VirtualEnvironmentType = str -LEGACY_VIRTUALENV = int(_virtualenv.__version__.split(".", 1)[0]) < 20 - - class VirtualEnvironment: """ An abstraction around virtual environments, currently it only uses @@ -34,7 +31,7 @@ def __init__( location: Path, template: Optional["VirtualEnvironment"] = None, venv_type: Optional[VirtualEnvironmentType] = None, - ): + ) -> None: self.location = location assert template is None or venv_type is None self._venv_type: VirtualEnvironmentType @@ -44,13 +41,18 @@ def __init__( self._venv_type = venv_type else: self._venv_type = "virtualenv" - assert self._venv_type in ("virtualenv", "venv") self._user_site_packages = False self._template = template self._sitecustomize: Optional[str] = None self._update_paths() self._create() + @property + def _legacy_virtualenv(self) -> bool: + if self._venv_type != "virtualenv": + return False + return int(_virtualenv.__version__.split(".", 1)[0]) < 20 + def __update_paths_legacy(self) -> None: home, lib, inc, bin = _virtualenv.path_locations(self.location) self.bin = Path(bin) @@ -63,7 +65,7 @@ def __update_paths_legacy(self) -> None: self.lib = Path(lib) def _update_paths(self) -> None: - if LEGACY_VIRTUALENV: + if self._legacy_virtualenv: self.__update_paths_legacy() return bases = { @@ -86,7 +88,11 @@ def _create(self, clear: bool = False) -> None: if self._template: # On Windows, calling `_virtualenv.path_locations(target)` # will have created the `target` directory... - if LEGACY_VIRTUALENV and sys.platform == "win32" and self.location.exists(): + if ( + self._legacy_virtualenv + and sys.platform == "win32" + and self.location.exists() + ): self.location.rmdir() # Clone virtual environment from template. shutil.copytree(self._template.location, self.location, symlinks=True) @@ -94,35 +100,36 @@ def _create(self, clear: bool = False) -> None: self._user_site_packages = self._template.user_site_packages else: # Create a new virtual environment. - if self._venv_type == "virtualenv": - if LEGACY_VIRTUALENV: - subprocess.check_call( - [ - sys.executable, - "-m", - "virtualenv", - "--no-pip", - "--no-wheel", - "--no-setuptools", - os.fspath(self.location), - ] - ) - self._fix_legacy_virtualenv_site_module() - else: - _virtualenv.cli_run( - [ - "--no-pip", - "--no-wheel", - "--no-setuptools", - os.fspath(self.location), - ], - ) + if self._legacy_virtualenv: + subprocess.check_call( + [ + sys.executable, + "-m", + "virtualenv", + "--no-pip", + "--no-wheel", + "--no-setuptools", + os.fspath(self.location), + ] + ) + self._fix_legacy_virtualenv_site_module() + elif self._venv_type == "virtualenv": + _virtualenv.cli_run( + [ + "--no-pip", + "--no-wheel", + "--no-setuptools", + os.fspath(self.location), + ], + ) elif self._venv_type == "venv": builder = _venv.EnvBuilder() context = builder.ensure_directories(self.location) builder.create_configuration(context) builder.setup_python(context) self.site.mkdir(parents=True, exist_ok=True) + else: + raise RuntimeError(f"Unsupported venv type {self._venv_type!r}") self.sitecustomize = self._sitecustomize self.user_site_packages = self._user_site_packages @@ -161,7 +168,9 @@ def _fix_legacy_virtualenv_site_module(self) -> None: assert compileall.compile_file(str(site_py), quiet=1, force=True) def _customize_site(self) -> None: - if not LEGACY_VIRTUALENV or self._venv_type == "venv": + if self._legacy_virtualenv: + contents = "" + else: # Enable user site (before system). contents = textwrap.dedent( f""" @@ -186,8 +195,6 @@ def _customize_site(self) -> None: site.addsitedir(path) """ ).strip() - else: - contents = "" if self._sitecustomize is not None: contents += "\n" + self._sitecustomize sitecustomize = self.site / "sitecustomize.py" @@ -234,14 +241,14 @@ def user_site_packages(self) -> bool: @user_site_packages.setter def user_site_packages(self, value: bool) -> None: self._user_site_packages = value - if not LEGACY_VIRTUALENV or self._venv_type == "venv": - self._rewrite_pyvenv_cfg( - {"include-system-site-packages": str(bool(value)).lower()} - ) - self._customize_site() - else: + if self._legacy_virtualenv: marker = self.lib / "no-global-site-packages.txt" if self._user_site_packages: marker.unlink() else: marker.touch() + else: + self._rewrite_pyvenv_cfg( + {"include-system-site-packages": str(bool(value)).lower()} + ) + self._customize_site()