[3.10] gh-92897: Ensure venv --copies respects source build property of th…

Lib/distutils/sysconfig.py

@@ -30,8 +30,6 @@
     parse_config_h as sysconfig_parse_config_h,
 
     _init_non_posix,
-    _is_python_source_dir,
-    _sys_home,
 
     _variable_rx,
     _findvar1_rx,
@@ -52,9 +50,6 @@
 # which might not be true in the time of import.
 _config_vars = get_config_vars()
 
-if os.name == "nt":
-    from sysconfig import _fix_pcbuild
-
 warnings.warn(
     'The distutils.sysconfig module is deprecated, use sysconfig instead',
     DeprecationWarning,
@@ -287,7 +282,7 @@ def get_python_inc(plat_specific=0, prefix=None):
             # must use "srcdir" from the makefile to find the "Include"
             # directory.
             if plat_specific:
-                return _sys_home or project_base
+                return project_base
             else:
                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
                 return os.path.normpath(incdir)

Lib/distutils/tests/test_sysconfig.py

@@ -60,7 +60,11 @@ def test_srcdir(self):
             # should be a full source checkout.
             Python_h = os.path.join(srcdir, 'Include', 'Python.h')
             self.assertTrue(os.path.exists(Python_h), Python_h)
-            self.assertTrue(sysconfig._is_python_source_dir(srcdir))
+            # <srcdir>/PC/pyconfig.h always exists even if unused on POSIX.
+            pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h')
+            self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h)
+            pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in')
+            self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in)
         elif os.name == 'posix':
             self.assertEqual(
                 os.path.dirname(sysconfig.get_makefile_filename()),

Lib/sysconfig.py

@@ -150,37 +150,38 @@ def _safe_realpath(path):
     # unable to retrieve the real program name
     _PROJECT_BASE = _safe_realpath(os.getcwd())
 
-if (os.name == 'nt' and
-    _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
-    _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir))
+# In a virtual environment, `sys._home` gives us the target directory
+# `_PROJECT_BASE` for the executable that created it when the virtual
+# python is an actual executable ('venv --copies' or Windows).
+_sys_home = getattr(sys, '_home', None)
+if _sys_home:
+    _PROJECT_BASE = _sys_home
+
+if os.name == 'nt':
+    # In a source build, the executable is in a subdirectory of the root
+    # that we want (<root>\PCbuild\<platname>).
+    # `_BASE_PREFIX` is used as the base installation is where the source
+    # will be.  The realpath is needed to prevent mount point confusion
+    # that can occur with just string comparisons.
+    if _safe_realpath(_PROJECT_BASE).startswith(
+            _safe_realpath(f'{_BASE_PREFIX}\\PCbuild')):
+        _PROJECT_BASE = _BASE_PREFIX
 
 # set for cross builds
 if "_PYTHON_PROJECT_BASE" in os.environ:
     _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"])
 
-def _is_python_source_dir(d):
+def is_python_build(check_home=None):
+    if check_home is not None:
+        import warnings
+        warnings.warn("check_home argument is deprecated and ignored.",
+                      DeprecationWarning, stacklevel=2)
     for fn in ("Setup", "Setup.local"):
-        if os.path.isfile(os.path.join(d, "Modules", fn)):
+        if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)):
             return True
     return False
 
-_sys_home = getattr(sys, '_home', None)
-
-if os.name == 'nt':
-    def _fix_pcbuild(d):
-        if d and os.path.normcase(d).startswith(
-                os.path.normcase(os.path.join(_PREFIX, "PCbuild"))):
-            return _PREFIX
-        return d
-    _PROJECT_BASE = _fix_pcbuild(_PROJECT_BASE)
-    _sys_home = _fix_pcbuild(_sys_home)
-
-def is_python_build(check_home=False):
-    if check_home and _sys_home:
-        return _is_python_source_dir(_sys_home)
-    return _is_python_source_dir(_PROJECT_BASE)
-
-_PYTHON_BUILD = is_python_build(True)
+_PYTHON_BUILD = is_python_build()
 
 if _PYTHON_BUILD:
     for scheme in ('posix_prefix', 'posix_home'):
@@ -389,7 +390,7 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True):
 def get_makefile_filename():
     """Return the path of the Makefile."""
     if _PYTHON_BUILD:
-        return os.path.join(_sys_home or _PROJECT_BASE, "Makefile")
+        return os.path.join(_PROJECT_BASE, "Makefile")
     if hasattr(sys, 'abiflags'):
         config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}'
     else:
@@ -534,9 +535,9 @@ def get_config_h_filename():
     """Return the path of pyconfig.h."""
     if _PYTHON_BUILD:
         if os.name == "nt":
-            inc_dir = os.path.join(_sys_home or _PROJECT_BASE, "PC")
+            inc_dir = os.path.join(_PROJECT_BASE, "PC")
         else:
-            inc_dir = _sys_home or _PROJECT_BASE
+            inc_dir = _PROJECT_BASE
     else:
         inc_dir = get_path('platinclude')
     return os.path.join(inc_dir, 'pyconfig.h')

Lib/test/test_sysconfig.py

@@ -376,7 +376,11 @@ def test_srcdir(self):
             # should be a full source checkout.
             Python_h = os.path.join(srcdir, 'Include', 'Python.h')
             self.assertTrue(os.path.exists(Python_h), Python_h)
-            self.assertTrue(sysconfig._is_python_source_dir(srcdir))
+            # <srcdir>/PC/pyconfig.h always exists even if unused on POSIX.
+            pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h')
+            self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h)
+            pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in')
+            self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in)
         elif os.name == 'posix':
             makefile_dir = os.path.dirname(sysconfig.get_makefile_filename())
             # Issue #19340: srcdir has been realpath'ed already

Lib/test/test_venv.py

@@ -9,18 +9,22 @@
 import ensurepip
 import os
 import os.path
+import pathlib
 import re
 import shutil
 import struct
 import subprocess
 import sys
+import sysconfig
 import tempfile
-from test.support import (captured_stdout, captured_stderr, requires_zlib,
-                          skip_if_broken_multiprocessing_synchronize)
+from test.support import (captured_stdout, captured_stderr,
+                          skip_if_broken_multiprocessing_synchronize, verbose,
+                          requires_subprocess, is_emscripten, is_wasi,
+                          requires_venv_with_pip)
 from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
 import unittest
 import venv
-from unittest.mock import patch
+from unittest.mock import patch, Mock
 
 try:
     import ctypes
@@ -34,13 +38,19 @@
     or sys._base_executable != sys.executable,
     'cannot run venv.create from within a venv on this platform')
 
+if is_emscripten or is_wasi:
+    raise unittest.SkipTest("venv is not available on Emscripten/WASI.")
+
+@requires_subprocess()
 def check_output(cmd, encoding=None):
     p = subprocess.Popen(cmd,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         encoding=encoding)
     out, err = p.communicate()
     if p.returncode:
+        if verbose and err:
+            print(err.decode('utf-8', 'backslashreplace'))
         raise subprocess.CalledProcessError(
             p.returncode, cmd, out, err)
     return out, err
@@ -92,12 +102,23 @@ def isdir(self, *args):
         fn = self.get_env_file(*args)
         self.assertTrue(os.path.isdir(fn))
 
-    def test_defaults(self):
+    def test_defaults_with_str_path(self):
         """
-        Test the create function with default arguments.
+        Test the create function with default arguments and a str path.
         """
         rmtree(self.env_dir)
         self.run_with_capture(venv.create, self.env_dir)
+        self._check_output_of_default_create()
+
+    def test_defaults_with_pathlib_path(self):
+        """
+        Test the create function with default arguments and a pathlib.Path path.
+        """
+        rmtree(self.env_dir)
+        self.run_with_capture(venv.create, pathlib.Path(self.env_dir))
+        self._check_output_of_default_create()
+
+    def _check_output_of_default_create(self):
         self.isdir(self.bindir)
         self.isdir(self.include)
         self.isdir(*self.lib)
@@ -113,13 +134,49 @@ def test_defaults(self):
         executable = sys._base_executable
         path = os.path.dirname(executable)
         self.assertIn('home = %s' % path, data)
+        self.assertIn('executable = %s' %
+                      os.path.realpath(sys.executable), data)
+        copies = '' if os.name=='nt' else ' --copies'
+        cmd = f'command = {sys.executable} -m venv{copies} --without-pip {self.env_dir}'
+        self.assertIn(cmd, data)
         fn = self.get_env_file(self.bindir, self.exe)
         if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
             bd = self.get_env_file(self.bindir)
             print('Contents of %r:' % bd)
             print('    %r' % os.listdir(bd))
         self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
 
+    def test_config_file_command_key(self):
+        attrs = [
+            (None, None),
+            ('symlinks', '--copies'),
+            ('with_pip', '--without-pip'),
+            ('system_site_packages', '--system-site-packages'),
+            ('clear', '--clear'),
+            ('upgrade', '--upgrade'),
+            ('upgrade_deps', '--upgrade-deps'),
+            ('prompt', '--prompt'),
+        ]
+        for attr, opt in attrs:
+            rmtree(self.env_dir)
+            if not attr:
+                b = venv.EnvBuilder()
+            else:
+                b = venv.EnvBuilder(
+                    **{attr: False if attr in ('with_pip', 'symlinks') else True})
+            b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps
+            b._setup_pip = Mock() # avoid pip setup
+            self.run_with_capture(b.create, self.env_dir)
+            data = self.get_text_file_contents('pyvenv.cfg')
+            if not attr:
+                for opt in ('--system-site-packages', '--clear', '--upgrade',
+                        '--upgrade-deps', '--prompt'):
+                    self.assertNotRegex(data, rf'command = .* {opt}')
+            elif os.name=='nt' and attr=='symlinks':
+                pass
+            else:
+                self.assertRegex(data, rf'command = .* {opt}')
+
     def test_prompt(self):
         env_name = os.path.split(self.env_dir)[1]
 
@@ -195,7 +252,52 @@ def test_prefixes(self):
             ('base_exec_prefix', sys.base_exec_prefix)):
             cmd[2] = 'import sys; print(sys.%s)' % prefix
             out, err = check_output(cmd)
-            self.assertEqual(out.strip(), expected.encode())
+            self.assertEqual(out.strip(), expected.encode(), prefix)
+
+    @requireVenvCreate
+    def test_sysconfig(self):
+        """
+        Test that the sysconfig functions work in a virtual environment.
+        """
+        rmtree(self.env_dir)
+        self.run_with_capture(venv.create, self.env_dir, symlinks=False)
+        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
+        cmd = [envpy, '-c', None]
+        for call, expected in (
+            # installation scheme
+            ('get_preferred_scheme("prefix")', 'venv'),
+            ('get_default_scheme()', 'venv'),
+            # build environment
+            ('is_python_build()', str(sysconfig.is_python_build())),
+            ('get_makefile_filename()', sysconfig.get_makefile_filename()),
+            ('get_config_h_filename()', sysconfig.get_config_h_filename())):
+            with self.subTest(call):
+                cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call
+                out, err = check_output(cmd)
+                self.assertEqual(out.strip(), expected.encode(), err)
+
+    @requireVenvCreate
+    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
+    def test_sysconfig_symlinks(self):
+        """
+        Test that the sysconfig functions work in a virtual environment.
+        """
+        rmtree(self.env_dir)
+        self.run_with_capture(venv.create, self.env_dir, symlinks=True)
+        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
+        cmd = [envpy, '-c', None]
+        for call, expected in (
+            # installation scheme
+            ('get_preferred_scheme("prefix")', 'venv'),
+            ('get_default_scheme()', 'venv'),
+            # build environment
+            ('is_python_build()', str(sysconfig.is_python_build())),
+            ('get_makefile_filename()', sysconfig.get_makefile_filename()),
+            ('get_config_h_filename()', sysconfig.get_config_h_filename())):
+            with self.subTest(call):
+                cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call
+                out, err = check_output(cmd)
+                self.assertEqual(out.strip(), expected.encode(), err)
 
     if sys.platform == 'win32':
         ENV_SUBDIRS = (
@@ -411,6 +513,16 @@ def test_macos_env(self):
             'import os; print("__PYVENV_LAUNCHER__" in os.environ)'])
         self.assertEqual(out.strip(), 'False'.encode())
 
+    def test_pathsep_error(self):
+        """
+        Test that venv creation fails when the target directory contains
+        the path separator.
+        """
+        rmtree(self.env_dir)
+        bad_itempath = self.env_dir + os.pathsep
+        self.assertRaises(ValueError, venv.create, bad_itempath)
+        self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath))
+
 @requireVenvCreate
 class EnsurePipTest(BaseTest):
     """Test venv module installation of pip."""
@@ -555,9 +667,7 @@ def nicer_error(self):
                 f"**Subprocess Error**\n{err}"
             )
 
-    # Issue #26610: pip/pep425tags.py requires ctypes
-    @unittest.skipUnless(ctypes, 'pip requires ctypes')
-    @requires_zlib()
+    @requires_venv_with_pip()
     def test_with_pip(self):
         self.do_test_with_pip(False)
         self.do_test_with_pip(True)