Test suite compatibility with the latest virtualenv >= 20.0.0

.azure-pipelines/steps/run-tests-windows.yml

@@ -24,7 +24,7 @@ steps:
     Set-Acl "R:\Temp" $acl
   displayName: Set RAMDisk Permissions
 
-- bash: pip install --upgrade 'virtualenv<20' setuptools tox
+- bash: pip install --upgrade 'virtualenv' setuptools tox
   displayName: Install Tox
 
 - script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml

.azure-pipelines/steps/run-tests.yml

@@ -4,7 +4,7 @@ steps:
   inputs:
     versionSpec: '$(python.version)'
 
-- bash: pip install --upgrade 'virtualenv<20' setuptools tox
+- bash: pip install --upgrade 'virtualenv' setuptools tox
   displayName: Install Tox
 
 - script: tox -e py -- -m unit -n auto --junit-xml=junit/unit-test.xml

src/pip/_internal/utils/virtualenv.py

@@ -4,7 +4,6 @@
 import logging
 import os
 import re
-import site
 import sys
 
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
@@ -90,30 +89,8 @@ def _no_global_under_venv():
     return False
 
 
-def _no_global_under_regular_virtualenv():
-    # type: () -> bool
-    """Check if "no-global-site-packages.txt" exists beside site.py
-
-    This mirrors logic in pypa/virtualenv for determining whether system
-    site-packages are visible in the virtual environment.
-    """
-    site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))
-    no_global_site_packages_file = os.path.join(
-        site_mod_dir, 'no-global-site-packages.txt',
-    )
-    return os.path.exists(no_global_site_packages_file)
-
-
 def virtualenv_no_global():
     # type: () -> bool
     """Returns a boolean, whether running in venv with no system site-packages.
     """
-    # PEP 405 compliance needs to be checked first since virtualenv >=20 would
-    # return True for both checks, but is only able to use the PEP 405 config.
-    if _running_under_venv():
-        return _no_global_under_venv()
-
-    if _running_under_regular_virtualenv():
-        return _no_global_under_regular_virtualenv()
-
-    return False
+    return _no_global_under_venv()

tests/functional/test_install.py

@@ -1589,7 +1589,9 @@ def test_install_compatible_python_requires(script):
 
 
 @pytest.mark.network
-def test_install_pep508_with_url(script):
+def test_install_pep508_with_url(script, virtualenv):
+    virtualenv.user_site_packages = False
+
     res = script.pip(
         'install', '--no-index',
         'packaging@https://files.pythonhosted.org/packages/2f/2b/'
@@ -1601,7 +1603,9 @@ def test_install_pep508_with_url(script):
 
 
 @pytest.mark.network
-def test_install_pep508_with_url_in_install_requires(script):
+def test_install_pep508_with_url_in_install_requires(script, virtualenv):
+    virtualenv.user_site_packages = False
+
     pkga_path = create_test_package_with_setup(
         script, name='pkga', version='1.0',
         install_requires=[

tests/functional/test_list.py

@@ -1,5 +1,6 @@
 import json
 import os
+import re
 
 import pytest
 
@@ -24,8 +25,8 @@ def test_basic_list(simple_script):
 
     """
     result = simple_script.pip('list')
-    assert 'simple     1.0' in result.stdout, str(result)
-    assert 'simple2    3.0' in result.stdout, str(result)
+    assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result)
+    assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result)
 
 
 def test_verbose_flag(simple_script):
@@ -37,8 +38,8 @@ def test_verbose_flag(simple_script):
     assert 'Version' in result.stdout, str(result)
     assert 'Location' in result.stdout, str(result)
     assert 'Installer' in result.stdout, str(result)
-    assert 'simple     1.0' in result.stdout, str(result)
-    assert 'simple2    3.0' in result.stdout, str(result)
+    assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result)
+    assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result)
 
 
 def test_columns_flag(simple_script):
@@ -49,8 +50,8 @@ def test_columns_flag(simple_script):
     assert 'Package' in result.stdout, str(result)
     assert 'Version' in result.stdout, str(result)
     assert 'simple (1.0)' not in result.stdout, str(result)
-    assert 'simple     1.0' in result.stdout, str(result)
-    assert 'simple2    3.0' in result.stdout, str(result)
+    assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result)
+    assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result)
 
 
 def test_format_priority(simple_script):
@@ -61,16 +62,16 @@ def test_format_priority(simple_script):
                                expect_stderr=True)
     assert 'simple==1.0' in result.stdout, str(result)
     assert 'simple2==3.0' in result.stdout, str(result)
-    assert 'simple     1.0' not in result.stdout, str(result)
-    assert 'simple2    3.0' not in result.stdout, str(result)
+    assert re.search(r'simple\s+1\.0', result.stdout) is None, str(result)
+    assert re.search(r'simple2\s+3\.0', result.stdout) is None, str(result)
 
     result = simple_script.pip('list', '--format=freeze', '--format=columns')
     assert 'Package' in result.stdout, str(result)
     assert 'Version' in result.stdout, str(result)
     assert 'simple==1.0' not in result.stdout, str(result)
     assert 'simple2==3.0' not in result.stdout, str(result)
-    assert 'simple     1.0' in result.stdout, str(result)
-    assert 'simple2    3.0' in result.stdout, str(result)
+    assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result)
+    assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result)
 
 
 def test_local_flag(simple_script):
@@ -91,7 +92,7 @@ def test_local_columns_flag(simple_script):
     assert 'Package' in result.stdout
     assert 'Version' in result.stdout
     assert 'simple (1.0)' not in result.stdout
-    assert 'simple     1.0' in result.stdout, str(result)
+    assert re.search(r'simple\s+1\.0', result.stdout) is not None, str(result)
 
 
 @pytest.mark.network
@@ -177,7 +178,7 @@ def test_uptodate_columns_flag(script, data):
     assert 'Location' in result.stdout      # editables included
     assert 'pip-test-package (0.1.1,' not in result.stdout
     assert 'pip-test-package 0.1.1' in result.stdout, str(result)
-    assert 'simple2          3.0' in result.stdout, str(result)
+    assert re.search(r'simple2\s+3\.0', result.stdout) is not None, str(result)
 
 
 @pytest.mark.network

tests/lib/venv.py

@@ -2,7 +2,7 @@
 
 import compileall
 import shutil
-import sys
+import sysconfig
 import textwrap
 
 import six
@@ -32,16 +32,15 @@ def __init__(self, location, template=None, venv_type=None):
         self._create()
 
     def _update_paths(self):
-        home, lib, inc, bin = _virtualenv.path_locations(self.location)
-        self.bin = Path(bin)
-        self.site = Path(lib) / 'site-packages'
-        # Workaround for https://github.com/pypa/virtualenv/issues/306
-        if hasattr(sys, "pypy_version_info"):
-            version_fmt = '{0}' if six.PY3 else '{0}.{1}'
-            version_dir = version_fmt.format(*sys.version_info)
-            self.lib = Path(home, 'lib-python', version_dir)
-        else:
-            self.lib = Path(lib)
+        paths = sysconfig.get_paths(vars={
+            "installed_base": self.location,
+            "installed_platbase": self.location,
+            "base": self.location,
+            "platbase": self.location,
+        })
+        self.bin = Path(paths["scripts"])
+        self.site = Path(paths["purelib"])
+        self.lib = Path(paths["stdlib"])
 
     def __repr__(self):
         return "<VirtualEnvironment {}>".format(self.location)
@@ -50,10 +49,6 @@ def _create(self, clear=False):
         if clear:
             shutil.rmtree(self.location)
         if self._template:
-            # On Windows, calling `_virtualenv.path_locations(target)`
-            # will have created the `target` directory...
-            if sys.platform == 'win32' and self.location.exists():
-                self.location.rmdir()
             # Clone virtual environment from template.
             shutil.copytree(
                 self._template.location, self.location, symlinks=True
@@ -63,88 +58,51 @@ def _create(self, clear=False):
         else:
             # Create a new virtual environment.
             if self._venv_type == 'virtualenv':
-                _virtualenv.create_environment(
+                _virtualenv.cli_run([
                     self.location,
-                    no_pip=True,
-                    no_wheel=True,
-                    no_setuptools=True,
-                )
-                self._fix_virtualenv_site_module()
+                    "--no-pip",
+                    "--no-wheel",
+                    "--no-setuptools",
+                ])
             elif self._venv_type == 'venv':
                 builder = _venv.EnvBuilder()
                 context = builder.ensure_directories(self.location)
                 builder.create_configuration(context)
                 builder.setup_python(context)
                 self.site.mkdir(parents=True, exist_ok=True)
+            else:
+                raise ValueError("venv type must be 'virtualenv' or 'venv'")
             self.sitecustomize = self._sitecustomize
             self.user_site_packages = self._user_site_packages
 
-    def _fix_virtualenv_site_module(self):
-        # Patch `site.py` so user site work as expected.
-        site_py = self.lib / 'site.py'
-        with open(site_py) as fp:
-            site_contents = fp.read()
-        for pattern, replace in (
-            (
-                # Ensure enabling user site does not result in adding
-                # the real site-packages' directory to `sys.path`.
-                (
-                    '\ndef virtual_addsitepackages(known_paths):\n'
-                ),
-                (
-                    '\ndef virtual_addsitepackages(known_paths):\n'
-                    '    return known_paths\n'
-                ),
-            ),
-            (
-                # Fix sites ordering: user site must be added before system.
-                (
-                    '\n    paths_in_sys = addsitepackages(paths_in_sys)'
-                    '\n    paths_in_sys = addusersitepackages(paths_in_sys)\n'
-                ),
-                (
-                    '\n    paths_in_sys = addusersitepackages(paths_in_sys)'
-                    '\n    paths_in_sys = addsitepackages(paths_in_sys)\n'
-                ),
-            ),
-        ):
-            assert pattern in site_contents
-            site_contents = site_contents.replace(pattern, replace)
-        with open(site_py, 'w') as fp:
-            fp.write(site_contents)
-        # Make sure bytecode is up-to-date too.
-        assert compileall.compile_file(str(site_py), quiet=1, force=True)
-
     def _customize_site(self):
-        contents = ''
-        if self._venv_type == 'venv':
-            # Enable user site (before system).
-            contents += textwrap.dedent(
-                '''
-                import os, site, sys
-
-                if not os.environ.get('PYTHONNOUSERSITE', False):
-
-                    site.ENABLE_USER_SITE = True
-
-                    # First, drop system-sites related paths.
-                    original_sys_path = sys.path[:]
-                    known_paths = set()
-                    for path in site.getsitepackages():
-                        site.addsitedir(path, known_paths=known_paths)
-                    system_paths = sys.path[len(original_sys_path):]
-                    for path in system_paths:
-                        if path in original_sys_path:
-                            original_sys_path.remove(path)
-                    sys.path = original_sys_path
-
-                    # Second, add user-site.
-                    site.addsitedir(site.getusersitepackages())
-
-                    # Third, add back system-sites related paths.
-                    for path in site.getsitepackages():
-                        site.addsitedir(path)
-                ''').strip()
+        # Enable user site (before system).
+        contents = textwrap.dedent(
+            '''
+            import os, site, sys
+
+            if not os.environ.get('PYTHONNOUSERSITE', False):
+
+                site.ENABLE_USER_SITE = True
+
+                # First, drop system-sites related paths.
+                original_sys_path = sys.path[:]
+                known_paths = set()
+                for path in site.getsitepackages():
+                    site.addsitedir(path, known_paths=known_paths)
+                system_paths = sys.path[len(original_sys_path):]
+                for path in system_paths:
+                    if path in original_sys_path:
+                        original_sys_path.remove(path)
+                sys.path = original_sys_path
+
+                # Second, add user-site.
+                site.addsitedir(site.getusersitepackages())
+
+                # Third, add back system-sites related paths.
+                for path in site.getsitepackages():
+                    site.addsitedir(path)
+            ''').strip()
         if self._sitecustomize is not None:
             contents += '\n' + self._sitecustomize
         sitecustomize = self.site / "sitecustomize.py"
@@ -176,11 +134,13 @@ def user_site_packages(self):
     @user_site_packages.setter
     def user_site_packages(self, value):
         self._user_site_packages = value
-        if self._venv_type == 'virtualenv':
-            marker = self.lib / "no-global-site-packages.txt"
-            if self._user_site_packages:
-                marker.unlink()
-            else:
-                marker.touch()
-        elif self._venv_type == 'venv':
-            self._customize_site()
+        self._customize_site()
+
+        pyvenv_cfg = self.location.joinpath("pyvenv.cfg")
+        modified_lines = []
+        for line in pyvenv_cfg.read_text().splitlines():
+            k, v = line.split("=", 1)
+            if k.strip() == "include-system-site-packages":
+                line = "{}= {}".format(k, "true" if value else "false")
+            modified_lines.append(line)
+        pyvenv_cfg.write_text("\n".join(modified_lines))

tests/unit/test_build_env.py

@@ -168,6 +168,8 @@ def test_build_env_overlay_prefix_has_priority(script):
     assert result.stdout.strip() == '2.0', str(result)
 
 
+@pytest.mark.skipif("sys.version_info < (3,)",
+                    reason="Incompatible with the latest virtualenv")
 @pytest.mark.incompatible_with_test_venv
 def test_build_env_isolation(script):