From 5e0626cfe4c502427ee9e2887e367f7215b60ff1 Mon Sep 17 00:00:00 2001 From: Jeremy Kloth Date: Tue, 5 Jul 2022 10:08:20 -0500 Subject: [PATCH] gh-92897: Ensure `venv --copies` respects source build property of the creating interpreter (GH-92899) (cherry picked from commit 067597522a9002f3b8aff7f46033f10acb2381e4) --- Lib/distutils/sysconfig.py | 7 +- Lib/distutils/tests/test_sysconfig.py | 6 +- Lib/sysconfig.py | 51 +++++----- Lib/test/test_sysconfig.py | 6 +- Lib/test/test_venv.py | 128 ++++++++++++++++++++++++-- 5 files changed, 156 insertions(+), 42 deletions(-) diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 3414a761e76b99..03b85584190db9 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -30,8 +30,6 @@ parse_config_h as sysconfig_parse_config_h, _init_non_posix, - _is_python_source_dir, - _sys_home, _variable_rx, _findvar1_rx, @@ -52,9 +50,6 @@ # which might not be true in the time of import. _config_vars = get_config_vars() -if os.name == "nt": - from sysconfig import _fix_pcbuild - warnings.warn( 'The distutils.sysconfig module is deprecated, use sysconfig instead', DeprecationWarning, @@ -287,7 +282,7 @@ def get_python_inc(plat_specific=0, prefix=None): # must use "srcdir" from the makefile to find the "Include" # directory. if plat_specific: - return _sys_home or project_base + return project_base else: incdir = os.path.join(get_config_var('srcdir'), 'Include') return os.path.normpath(incdir) diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py index 59676b0e0b0ce2..040c2c5e81e3b5 100644 --- a/Lib/distutils/tests/test_sysconfig.py +++ b/Lib/distutils/tests/test_sysconfig.py @@ -60,7 +60,11 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - self.assertTrue(sysconfig._is_python_source_dir(srcdir)) + # /PC/pyconfig.h always exists even if unused on POSIX. + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) + pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') + self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) elif os.name == 'posix': self.assertEqual( os.path.dirname(sysconfig.get_makefile_filename()), diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index daf9f000060a35..6dd2a9bbb6a909 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -150,37 +150,38 @@ def _safe_realpath(path): # unable to retrieve the real program name _PROJECT_BASE = _safe_realpath(os.getcwd()) -if (os.name == 'nt' and - _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): - _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) +# In a virtual environment, `sys._home` gives us the target directory +# `_PROJECT_BASE` for the executable that created it when the virtual +# python is an actual executable ('venv --copies' or Windows). +_sys_home = getattr(sys, '_home', None) +if _sys_home: + _PROJECT_BASE = _sys_home + +if os.name == 'nt': + # In a source build, the executable is in a subdirectory of the root + # that we want (\PCbuild\). + # `_BASE_PREFIX` is used as the base installation is where the source + # will be. The realpath is needed to prevent mount point confusion + # that can occur with just string comparisons. + if _safe_realpath(_PROJECT_BASE).startswith( + _safe_realpath(f'{_BASE_PREFIX}\\PCbuild')): + _PROJECT_BASE = _BASE_PREFIX # set for cross builds if "_PYTHON_PROJECT_BASE" in os.environ: _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) -def _is_python_source_dir(d): +def is_python_build(check_home=None): + if check_home is not None: + import warnings + warnings.warn("check_home argument is deprecated and ignored.", + DeprecationWarning, stacklevel=2) for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): + if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): return True return False -_sys_home = getattr(sys, '_home', None) - -if os.name == 'nt': - def _fix_pcbuild(d): - if d and os.path.normcase(d).startswith( - os.path.normcase(os.path.join(_PREFIX, "PCbuild"))): - return _PREFIX - return d - _PROJECT_BASE = _fix_pcbuild(_PROJECT_BASE) - _sys_home = _fix_pcbuild(_sys_home) - -def is_python_build(check_home=False): - if check_home and _sys_home: - return _is_python_source_dir(_sys_home) - return _is_python_source_dir(_PROJECT_BASE) - -_PYTHON_BUILD = is_python_build(True) +_PYTHON_BUILD = is_python_build() if _PYTHON_BUILD: for scheme in ('posix_prefix', 'posix_home'): @@ -389,7 +390,7 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: - return os.path.join(_sys_home or _PROJECT_BASE, "Makefile") + return os.path.join(_PROJECT_BASE, "Makefile") if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: @@ -534,9 +535,9 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_sys_home or _PROJECT_BASE, "PC") + inc_dir = os.path.join(_PROJECT_BASE, "PC") else: - inc_dir = _sys_home or _PROJECT_BASE + inc_dir = _PROJECT_BASE else: inc_dir = get_path('platinclude') return os.path.join(inc_dir, 'pyconfig.h') diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 5ee9839c0487d1..706ba951a71b28 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -376,7 +376,11 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - self.assertTrue(sysconfig._is_python_source_dir(srcdir)) + # /PC/pyconfig.h always exists even if unused on POSIX. + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) + pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') + self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index eca35ec4bfd1ab..4359a4e3ebe861 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -9,18 +9,22 @@ import ensurepip import os import os.path +import pathlib import re import shutil import struct import subprocess import sys +import sysconfig import tempfile -from test.support import (captured_stdout, captured_stderr, requires_zlib, - skip_if_broken_multiprocessing_synchronize) +from test.support import (captured_stdout, captured_stderr, + skip_if_broken_multiprocessing_synchronize, verbose, + requires_subprocess, is_emscripten, is_wasi, + requires_venv_with_pip) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv -from unittest.mock import patch +from unittest.mock import patch, Mock try: import ctypes @@ -34,6 +38,10 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') +if is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on Emscripten/WASI.") + +@requires_subprocess() def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, @@ -41,6 +49,8 @@ def check_output(cmd, encoding=None): encoding=encoding) out, err = p.communicate() if p.returncode: + if verbose and err: + print(err.decode('utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) return out, err @@ -92,12 +102,23 @@ def isdir(self, *args): fn = self.get_env_file(*args) self.assertTrue(os.path.isdir(fn)) - def test_defaults(self): + def test_defaults_with_str_path(self): """ - Test the create function with default arguments. + Test the create function with default arguments and a str path. """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) + self._check_output_of_default_create() + + def test_defaults_with_pathlib_path(self): + """ + Test the create function with default arguments and a pathlib.Path path. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, pathlib.Path(self.env_dir)) + self._check_output_of_default_create() + + def _check_output_of_default_create(self): self.isdir(self.bindir) self.isdir(self.include) self.isdir(*self.lib) @@ -113,6 +134,11 @@ def test_defaults(self): executable = sys._base_executable path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) + self.assertIn('executable = %s' % + os.path.realpath(sys.executable), data) + copies = '' if os.name=='nt' else ' --copies' + cmd = f'command = {sys.executable} -m venv{copies} --without-pip {self.env_dir}' + self.assertIn(cmd, data) fn = self.get_env_file(self.bindir, self.exe) if not os.path.exists(fn): # diagnostics for Windows buildbot failures bd = self.get_env_file(self.bindir) @@ -120,6 +146,37 @@ def test_defaults(self): print(' %r' % os.listdir(bd)) self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) + def test_config_file_command_key(self): + attrs = [ + (None, None), + ('symlinks', '--copies'), + ('with_pip', '--without-pip'), + ('system_site_packages', '--system-site-packages'), + ('clear', '--clear'), + ('upgrade', '--upgrade'), + ('upgrade_deps', '--upgrade-deps'), + ('prompt', '--prompt'), + ] + for attr, opt in attrs: + rmtree(self.env_dir) + if not attr: + b = venv.EnvBuilder() + else: + b = venv.EnvBuilder( + **{attr: False if attr in ('with_pip', 'symlinks') else True}) + b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps + b._setup_pip = Mock() # avoid pip setup + self.run_with_capture(b.create, self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + if not attr: + for opt in ('--system-site-packages', '--clear', '--upgrade', + '--upgrade-deps', '--prompt'): + self.assertNotRegex(data, rf'command = .* {opt}') + elif os.name=='nt' and attr=='symlinks': + pass + else: + self.assertRegex(data, rf'command = .* {opt}') + def test_prompt(self): env_name = os.path.split(self.env_dir)[1] @@ -195,7 +252,52 @@ def test_prefixes(self): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode()) + self.assertEqual(out.strip(), expected.encode(), prefix) + + @requireVenvCreate + def test_sysconfig(self): + """ + Test that the sysconfig functions work in a virtual environment. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, symlinks=False) + envpy = os.path.join(self.env_dir, self.bindir, self.exe) + cmd = [envpy, '-c', None] + for call, expected in ( + # installation scheme + ('get_preferred_scheme("prefix")', 'venv'), + ('get_default_scheme()', 'venv'), + # build environment + ('is_python_build()', str(sysconfig.is_python_build())), + ('get_makefile_filename()', sysconfig.get_makefile_filename()), + ('get_config_h_filename()', sysconfig.get_config_h_filename())): + with self.subTest(call): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd) + self.assertEqual(out.strip(), expected.encode(), err) + + @requireVenvCreate + @unittest.skipUnless(can_symlink(), 'Needs symlinks') + def test_sysconfig_symlinks(self): + """ + Test that the sysconfig functions work in a virtual environment. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, symlinks=True) + envpy = os.path.join(self.env_dir, self.bindir, self.exe) + cmd = [envpy, '-c', None] + for call, expected in ( + # installation scheme + ('get_preferred_scheme("prefix")', 'venv'), + ('get_default_scheme()', 'venv'), + # build environment + ('is_python_build()', str(sysconfig.is_python_build())), + ('get_makefile_filename()', sysconfig.get_makefile_filename()), + ('get_config_h_filename()', sysconfig.get_config_h_filename())): + with self.subTest(call): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd) + self.assertEqual(out.strip(), expected.encode(), err) if sys.platform == 'win32': ENV_SUBDIRS = ( @@ -411,6 +513,16 @@ def test_macos_env(self): 'import os; print("__PYVENV_LAUNCHER__" in os.environ)']) self.assertEqual(out.strip(), 'False'.encode()) + def test_pathsep_error(self): + """ + Test that venv creation fails when the target directory contains + the path separator. + """ + rmtree(self.env_dir) + bad_itempath = self.env_dir + os.pathsep + self.assertRaises(ValueError, venv.create, bad_itempath) + self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath)) + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" @@ -555,9 +667,7 @@ def nicer_error(self): f"**Subprocess Error**\n{err}" ) - # Issue #26610: pip/pep425tags.py requires ctypes - @unittest.skipUnless(ctypes, 'pip requires ctypes') - @requires_zlib() + @requires_venv_with_pip() def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True)