diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d555c53fee50a2..7f8b1d71dbd227 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1865,15 +1865,16 @@ def missing_compiler_executable(cmd_names=[]): missing. """ - # TODO (PEP 632): alternate check without using distutils - from distutils import ccompiler, sysconfig, spawn, errors + from setuptools._distutils import ccompiler, sysconfig, spawn + from setuptools import errors + compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) if compiler.compiler_type == "msvc": # MSVC has no executables, so check whether initialization succeeds try: compiler.initialize() - except errors.DistutilsPlatformError: + except errors.PlatformError: return "msvc" for name in compiler.executables: if cmd_names and name not in cmd_names: @@ -2270,6 +2271,42 @@ def requires_venv_with_pip(): return unittest.skipUnless(ctypes, 'venv: pip requires ctypes') +# Context manager that creates a virtual environment, install setuptools and wheel in it +# and returns the path to the venv directory and the path to the python executable +@contextlib.contextmanager +def setup_venv_with_pip_setuptools_wheel(venv_dir): + import subprocess + from .os_helper import temp_cwd + + with temp_cwd() as temp_dir: + # Create virtual environment to get setuptools + cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] + if verbose: + print() + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True) + + venv = os.path.join(temp_dir, venv_dir) + + # Get the Python executable of the venv + python_exe = os.path.basename(sys.executable) + if sys.platform == 'win32': + python = os.path.join(venv, 'Scripts', python_exe) + else: + python = os.path.join(venv, 'bin', python_exe) + + cmd = [python, '-X', 'dev', + '-m', 'pip', 'install', + findfile('setuptools-67.6.1-py3-none-any.whl'), + findfile('wheel-0.40.0-py3-none-any.whl')] + if verbose: + print() + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True) + + yield python + + # True if Python is built with the Py_DEBUG macro defined: if # Python is built in debug mode (./configure --with-pydebug). Py_DEBUG = hasattr(sys, 'gettotalrefcount') diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext.py index 4fb62d87e860fc..e2fedc9735079f 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext.py @@ -35,39 +35,20 @@ def test_build_cpp03(self): # the test uses venv+pip: skip if it's not available @support.requires_venv_with_pip() def check_build(self, std_cpp03, extension_name): - # Build in a temporary directory - with os_helper.temp_cwd(): - self._check_build(std_cpp03, extension_name) + venv_dir = 'env' + with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: + self._check_build(std_cpp03, extension_name, python_exe) - def _check_build(self, std_cpp03, extension_name): + def _check_build(self, std_cpp03, extension_name, python_exe): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py")) - venv_dir = 'env' - verbose = support.verbose - - # Create virtual environment to get setuptools - cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) - - # Get the Python executable of the venv - python_exe = 'python' - if sys.executable.endswith('.exe'): - python_exe += '.exe' - if MS_WINDOWS: - python = os.path.join(venv_dir, 'Scripts', python_exe) - else: - python = os.path.join(venv_dir, 'bin', python_exe) - def run_cmd(operation, cmd): env = os.environ.copy() env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11' env['CPYTHON_TEST_EXT_NAME'] = extension_name - if verbose: + if support.verbose: print('Run:', ' '.join(cmd)) subprocess.run(cmd, check=True, env=env) else: @@ -81,14 +62,8 @@ def run_cmd(operation, cmd): self.fail( f"{operation} failed with exit code {proc.returncode}") - cmd = [python, '-X', 'dev', - '-m', 'pip', 'install', - support.findfile('setuptools-67.6.1-py3-none-any.whl'), - support.findfile('wheel-0.40.0-py3-none-any.whl')] - run_cmd('Install build dependencies', cmd) - # Build and install the C++ extension - cmd = [python, '-X', 'dev', + cmd = [python_exe, '-X', 'dev', '-m', 'pip', 'install', '--no-build-isolation', os.path.abspath(pkg_dir)] run_cmd('Install', cmd) @@ -96,14 +71,14 @@ def run_cmd(operation, cmd): # Do a reference run. Until we test that running python # doesn't leak references (gh-94755), run it so one can manually check # -X showrefcount results against this baseline. - cmd = [python, + cmd = [python_exe, '-X', 'dev', '-X', 'showrefcount', '-c', 'pass'] run_cmd('Reference run', cmd) # Import the C++ extension - cmd = [python, + cmd = [python_exe, '-X', 'dev', '-X', 'showrefcount', '-c', f"import {extension_name}"] diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py index 7c402c3d7c5acf..77f72fcc7c6e3b 100644 --- a/Lib/test/test_peg_generator/__init__.py +++ b/Lib/test/test_peg_generator/__init__.py @@ -3,9 +3,6 @@ from test import support from test.support import load_package_tests -# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12 -raise unittest.SkipTest("distutils has been removed in Python 3.12") - if support.check_sanitizer(address=True, memory=True): # bpo-46633: Skip the test because it is too slow when Python is built diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index d34ffef0dbc5ec..af39faeba94357 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -1,3 +1,5 @@ +import contextlib +import subprocess import sysconfig import textwrap import unittest @@ -8,7 +10,7 @@ from test import test_tools from test import support -from test.support import os_helper +from test.support import os_helper, import_helper from test.support.script_helper import assert_python_ok _py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST") @@ -88,6 +90,16 @@ def setUpClass(cls): cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base) cls.addClassCleanup(shutil.rmtree, cls.library_dir) + with contextlib.ExitStack() as stack: + python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv")) + sitepackages = subprocess.check_output( + [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], + text=True, + ).strip() + stack.enter_context(import_helper.DirsOnSysPath(sitepackages)) + cls.addClassCleanup(stack.pop_all().close) + + @support.requires_venv_with_pip() def setUp(self): self._backup_config_vars = dict(sysconfig._CONFIG_VARS) cmd = support.missing_compiler_executable() diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 30e992ed213c67..2122242e9a8e8a 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -96,14 +96,14 @@ def test_gather(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2" ), TokenInfo( - NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n" + NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n" + NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2" ), ], ) @@ -119,8 +119,8 @@ def test_expr_grammar(self) -> None: self.assertEqual( node, [ - TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n"), - TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42"), + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42"), ], ) @@ -137,19 +137,19 @@ def test_optional_operator(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2" ), [ TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2" ), TokenInfo( - NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n" + NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2" ), ], ], TokenInfo( - NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n" + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2" ), ], ) @@ -158,10 +158,10 @@ def test_optional_operator(self) -> None: node, [ [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"), None, ], - TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"), ], ) @@ -178,11 +178,11 @@ def test_optional_literal(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+" ), - TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+\n"), + TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+"), ], - TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+"), ], ) node = parse_string("1\n", parser_class) @@ -190,10 +190,10 @@ def test_optional_literal(self) -> None: node, [ [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"), None, ], - TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"), ], ) @@ -210,19 +210,19 @@ def test_alt_optional_operator(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2" ), [ TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2" ), TokenInfo( - NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n" + NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2" ), ], ], TokenInfo( - NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n" + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2" ), ], ) @@ -231,10 +231,10 @@ def test_alt_optional_operator(self) -> None: node, [ [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"), None, ], - TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"), ], ) @@ -248,17 +248,17 @@ def test_repeat_0_simple(self) -> None: self.assertEqual( node, [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3"), [ TokenInfo( - NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n" + NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3" ), TokenInfo( - NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n" + NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n" + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3" ), ], ) @@ -266,9 +266,9 @@ def test_repeat_0_simple(self) -> None: self.assertEqual( node, [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"), [], - TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"), ], ) @@ -283,36 +283,36 @@ def test_repeat_0_complex(self) -> None: node, [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3" ), [ [ TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="2", start=(1, 4), end=(1, 5), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], [ TokenInfo( - OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="3", start=(1, 8), end=(1, 9), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], ], TokenInfo( - NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3" ), ], ) @@ -327,17 +327,17 @@ def test_repeat_1_simple(self) -> None: self.assertEqual( node, [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3"), [ TokenInfo( - NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n" + NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3" ), TokenInfo( - NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n" + NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n" + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3" ), ], ) @@ -355,36 +355,36 @@ def test_repeat_1_complex(self) -> None: node, [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3" ), [ [ TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="2", start=(1, 4), end=(1, 5), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], [ TokenInfo( - OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="3", start=(1, 8), end=(1, 9), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], ], TokenInfo( - NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3" ), ], ) @@ -403,17 +403,17 @@ def test_repeat_with_sep_simple(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3" ), TokenInfo( - NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3\n" + NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3" ), TokenInfo( - NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3\n" + NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3\n" + NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3" ), ], ) @@ -447,28 +447,28 @@ def test_left_recursive(self) -> None: string="1", start=(1, 0), end=(1, 1), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="2", start=(1, 4), end=(1, 5), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], TokenInfo( - OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3" ), TokenInfo( - NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n" + NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3" ), ], ) @@ -794,7 +794,7 @@ def test_soft_keyword(self) -> None: start: | "number" n=NUMBER { eval(n.string) } | "string" n=STRING { n.string } - | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"} + | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string } """ parser_class = make_parser(grammar) self.assertEqual(parse_string("number 1", parser_class), 1) diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 5805ff63717440..aace684045b9f8 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -1,4 +1,5 @@ import itertools +import os import pathlib import sys import sysconfig @@ -27,6 +28,46 @@ def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[ return f"{flags} {py_flags_nodist}".split() +def fixup_build_ext(cmd): + """Function needed to make build_ext tests pass. + + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. + + When Python was built with in debug mode on Windows, build_ext commands + need their debug attribute set, and it is not done automatically for + some reason. + + This function handles both of these things. Example use: + + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + + Unlike most other Unix platforms, Mac OS X embeds absolute paths + to shared libraries into executables, so the fixup is not needed there. + + Taken from distutils (was part of the CPython stdlib until Python 3.11) + """ + if os.name == 'nt': + cmd.debug = sys.executable.endswith('_d.exe') + elif sysconfig.get_config_var('Py_ENABLE_SHARED'): + # To further add to the shared builds fun on Unix, we can't just add + # library_dirs to the Extension() instance because that doesn't get + # plumbed through to the final compiler command. + runshared = sysconfig.get_config_var('RUNSHARED') + if runshared is None: + cmd.library_dirs = ['.'] + else: + if sys.platform == 'darwin': + cmd.library_dirs = [] + else: + name, equals, value = runshared.partition('=') + cmd.library_dirs = [d for d in value.split(os.pathsep) if d] + + + def compile_c_extension( generated_source_path: str, build_dir: Optional[str] = None, @@ -49,16 +90,15 @@ def compile_c_extension( static library of the common parser sources (this is useful in case you are creating multiple extensions). """ - import distutils.log - from distutils.core import Distribution, Extension - from distutils.tests.support import fixup_build_ext # type: ignore + import setuptools.logging - from distutils.ccompiler import new_compiler - from distutils.dep_util import newer_group - from distutils.sysconfig import customize_compiler + from setuptools import Extension, Distribution + from setuptools._distutils.dep_util import newer_group + from setuptools._distutils.ccompiler import new_compiler + from setuptools._distutils.sysconfig import customize_compiler if verbose: - distutils.log.set_threshold(distutils.log.DEBUG) + setuptools.logging.set_threshold(setuptools.logging.logging.DEBUG) source_file_path = pathlib.Path(generated_source_path) extension_name = source_file_path.stem