From 88edfa75caffd3e608d9aae6b7c305c2eabea3e9 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 23 May 2023 16:20:23 +0200 Subject: [PATCH 1/8] gh-100238: Use setuptools in peg-generator and reenable tests --- Lib/test/support/__init__.py | 40 ++++++++++++++++++-- Lib/test/test_cppext.py | 35 +++-------------- Lib/test/test_peg_generator/__init__.py | 3 -- Lib/test/test_peg_generator/test_c_parser.py | 12 +++++- Lib/test/test_peg_generator/test_pegen.py | 36 +++++++++--------- Tools/peg_generator/pegen/build.py | 14 +++---- 6 files changed, 77 insertions(+), 63 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d555c53fee50a2..aa3518c0da427a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1865,15 +1865,16 @@ def missing_compiler_executable(cmd_names=[]): missing. """ - # TODO (PEP 632): alternate check without using distutils - from distutils import ccompiler, sysconfig, spawn, errors + from setuptools._distutils import ccompiler, sysconfig, spawn + from setuptools import errors + compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) if compiler.compiler_type == "msvc": # MSVC has no executables, so check whether initialization succeeds try: compiler.initialize() - except errors.DistutilsPlatformError: + except errors.PlatformError: return "msvc" for name in compiler.executables: if cmd_names and name not in cmd_names: @@ -2270,6 +2271,39 @@ def requires_venv_with_pip(): return unittest.skipUnless(ctypes, 'venv: pip requires ctypes') +# Context manager that creates a virtual environment, install setuptools and wheel in it +# and returns the path to the venv directory and the path to the python executable +@contextlib.contextmanager +def setup_venv_with_pip_setuptools_wheel(venv_dir): + import subprocess + from .os_helper import temp_cwd + + with temp_cwd() as temp_dir: + # Create virtual environment to get setuptools + cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] + if verbose: + print() + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True) + + # Get the Python executable of the venv + python_exe = 'python' + if sys.executable.endswith('.exe'): + python_exe += '.exe' + if sys.platform == 'win32': + python = os.path.join(venv_dir, 'Scripts', python_exe) + else: + python = os.path.join(venv_dir, 'bin', python_exe) + + cmd = [python, '-X', 'dev', + '-m', 'pip', 'install', + findfile('setuptools-67.6.1-py3-none-any.whl'), + findfile('wheel-0.40.0-py3-none-any.whl')] + subprocess.run(cmd, check=True) + + yield os.path.join(temp_dir, venv_dir), python + + # True if Python is built with the Py_DEBUG macro defined: if # Python is built in debug mode (./configure --with-pydebug). Py_DEBUG = hasattr(sys, 'gettotalrefcount') diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext.py index 4fb62d87e860fc..d3dd203b43a52a 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext.py @@ -35,39 +35,20 @@ def test_build_cpp03(self): # the test uses venv+pip: skip if it's not available @support.requires_venv_with_pip() def check_build(self, std_cpp03, extension_name): - # Build in a temporary directory - with os_helper.temp_cwd(): - self._check_build(std_cpp03, extension_name) + venv_dir = 'env' + with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as (_, python): + self._check_build(std_cpp03, extension_name, python) - def _check_build(self, std_cpp03, extension_name): + def _check_build(self, std_cpp03, extension_name, python): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py")) - venv_dir = 'env' - verbose = support.verbose - - # Create virtual environment to get setuptools - cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) - - # Get the Python executable of the venv - python_exe = 'python' - if sys.executable.endswith('.exe'): - python_exe += '.exe' - if MS_WINDOWS: - python = os.path.join(venv_dir, 'Scripts', python_exe) - else: - python = os.path.join(venv_dir, 'bin', python_exe) - def run_cmd(operation, cmd): env = os.environ.copy() env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11' env['CPYTHON_TEST_EXT_NAME'] = extension_name - if verbose: + if support.verbose: print('Run:', ' '.join(cmd)) subprocess.run(cmd, check=True, env=env) else: @@ -81,12 +62,6 @@ def run_cmd(operation, cmd): self.fail( f"{operation} failed with exit code {proc.returncode}") - cmd = [python, '-X', 'dev', - '-m', 'pip', 'install', - support.findfile('setuptools-67.6.1-py3-none-any.whl'), - support.findfile('wheel-0.40.0-py3-none-any.whl')] - run_cmd('Install build dependencies', cmd) - # Build and install the C++ extension cmd = [python, '-X', 'dev', '-m', 'pip', 'install', '--no-build-isolation', diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py index 7c402c3d7c5acf..77f72fcc7c6e3b 100644 --- a/Lib/test/test_peg_generator/__init__.py +++ b/Lib/test/test_peg_generator/__init__.py @@ -3,9 +3,6 @@ from test import support from test.support import load_package_tests -# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12 -raise unittest.SkipTest("distutils has been removed in Python 3.12") - if support.check_sanitizer(address=True, memory=True): # bpo-46633: Skip the test because it is too slow when Python is built diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index d34ffef0dbc5ec..34c3009002fb80 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -1,3 +1,4 @@ +import contextlib import sysconfig import textwrap import unittest @@ -8,7 +9,7 @@ from test import test_tools from test import support -from test.support import os_helper +from test.support import os_helper, import_helper from test.support.script_helper import assert_python_ok _py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST") @@ -88,6 +89,15 @@ def setUpClass(cls): cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base) cls.addClassCleanup(shutil.rmtree, cls.library_dir) + with contextlib.ExitStack() as stack: + full_venv_path, _ = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv")) + stack.enter_context(import_helper.DirsOnSysPath(os.path.join(full_venv_path, + "lib", + f"python{sysconfig.get_python_version()}", + "site-packages"))) + cls.addClassCleanup(stack.pop_all().close) + + @support.requires_venv_with_pip() def setUp(self): self._backup_config_vars = dict(sysconfig._CONFIG_VARS) cmd = support.missing_compiler_executable() diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 30e992ed213c67..bad0b1d2f53796 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -552,14 +552,14 @@ def test_mutually_left_recursive(self) -> None: string="D", start=(1, 0), end=(1, 1), - line="D A C A E", + line="D A C A E\n", ), TokenInfo( type=NAME, string="A", start=(1, 2), end=(1, 3), - line="D A C A E", + line="D A C A E\n", ), ], TokenInfo( @@ -567,7 +567,7 @@ def test_mutually_left_recursive(self) -> None: string="C", start=(1, 4), end=(1, 5), - line="D A C A E", + line="D A C A E\n", ), ], TokenInfo( @@ -575,11 +575,11 @@ def test_mutually_left_recursive(self) -> None: string="A", start=(1, 6), end=(1, 7), - line="D A C A E", + line="D A C A E\n", ), ], TokenInfo( - type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E" + type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E\n" ), ], ) @@ -594,22 +594,22 @@ def test_mutually_left_recursive(self) -> None: string="B", start=(1, 0), end=(1, 1), - line="B C A E", + line="B C A E\n", ), TokenInfo( type=NAME, string="C", start=(1, 2), end=(1, 3), - line="B C A E", + line="B C A E\n", ), ], TokenInfo( - type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E" + type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E\n" ), ], TokenInfo( - type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E" + type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E\n" ), ], ) @@ -654,10 +654,10 @@ def test_lookahead(self) -> None: node, [ TokenInfo( - NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 ." + NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 .\n" ), TokenInfo( - OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 ." + OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 .\n" ), [ TokenInfo( @@ -665,7 +665,7 @@ def test_lookahead(self) -> None: string="12", start=(1, 6), end=(1, 8), - line="foo = 12 + 12 .", + line="foo = 12 + 12 .\n", ), [ [ @@ -674,14 +674,14 @@ def test_lookahead(self) -> None: string="+", start=(1, 9), end=(1, 10), - line="foo = 12 + 12 .", + line="foo = 12 + 12 .\n", ), TokenInfo( NUMBER, string="12", start=(1, 11), end=(1, 13), - line="foo = 12 + 12 .", + line="foo = 12 + 12 .\n", ), ] ], @@ -733,9 +733,9 @@ def test_cut(self) -> None: self.assertEqual( node, [ - TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"), - TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)"), - TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"), + TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)\n"), + TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)\n"), + TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)\n"), ], ) @@ -794,7 +794,7 @@ def test_soft_keyword(self) -> None: start: | "number" n=NUMBER { eval(n.string) } | "string" n=STRING { n.string } - | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"} + | SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string } """ parser_class = make_parser(grammar) self.assertEqual(parse_string("number 1", parser_class), 1) diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 5805ff63717440..2d50f87f198d85 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -49,16 +49,15 @@ def compile_c_extension( static library of the common parser sources (this is useful in case you are creating multiple extensions). """ - import distutils.log - from distutils.core import Distribution, Extension - from distutils.tests.support import fixup_build_ext # type: ignore + import setuptools.logging - from distutils.ccompiler import new_compiler - from distutils.dep_util import newer_group - from distutils.sysconfig import customize_compiler + from setuptools import Extension, Distribution + from setuptools._distutils.dep_util import newer_group + from setuptools._distutils.ccompiler import new_compiler + from setuptools._distutils.sysconfig import customize_compiler if verbose: - distutils.log.set_threshold(distutils.log.DEBUG) + setuptools.logging.set_threshold(setuptools.logging.logging.DEBUG) source_file_path = pathlib.Path(generated_source_path) extension_name = source_file_path.stem @@ -100,7 +99,6 @@ def compile_c_extension( ) dist = Distribution({"name": extension_name, "ext_modules": [extension]}) cmd = dist.get_command_obj("build_ext") - fixup_build_ext(cmd) cmd.build_lib = str(source_file_path.parent) cmd.include_dirs = include_dirs if build_dir: From 907eae84d0d2d11f48a8ed7057959cafd250bb3e Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 24 May 2023 13:15:28 +0200 Subject: [PATCH 2/8] Roll back changes re. newline in tokenizer --- Lib/test/test_peg_generator/test_pegen.py | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index bad0b1d2f53796..d92da7b29bff98 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -552,14 +552,14 @@ def test_mutually_left_recursive(self) -> None: string="D", start=(1, 0), end=(1, 1), - line="D A C A E\n", + line="D A C A E", ), TokenInfo( type=NAME, string="A", start=(1, 2), end=(1, 3), - line="D A C A E\n", + line="D A C A E", ), ], TokenInfo( @@ -567,7 +567,7 @@ def test_mutually_left_recursive(self) -> None: string="C", start=(1, 4), end=(1, 5), - line="D A C A E\n", + line="D A C A E", ), ], TokenInfo( @@ -575,11 +575,11 @@ def test_mutually_left_recursive(self) -> None: string="A", start=(1, 6), end=(1, 7), - line="D A C A E\n", + line="D A C A E", ), ], TokenInfo( - type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E\n" + type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E" ), ], ) @@ -594,22 +594,22 @@ def test_mutually_left_recursive(self) -> None: string="B", start=(1, 0), end=(1, 1), - line="B C A E\n", + line="B C A E", ), TokenInfo( type=NAME, string="C", start=(1, 2), end=(1, 3), - line="B C A E\n", + line="B C A E", ), ], TokenInfo( - type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E\n" + type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E" ), ], TokenInfo( - type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E\n" + type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E" ), ], ) @@ -654,10 +654,10 @@ def test_lookahead(self) -> None: node, [ TokenInfo( - NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 .\n" + NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 ." ), TokenInfo( - OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 .\n" + OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 ." ), [ TokenInfo( @@ -665,7 +665,7 @@ def test_lookahead(self) -> None: string="12", start=(1, 6), end=(1, 8), - line="foo = 12 + 12 .\n", + line="foo = 12 + 12 .", ), [ [ @@ -674,14 +674,14 @@ def test_lookahead(self) -> None: string="+", start=(1, 9), end=(1, 10), - line="foo = 12 + 12 .\n", + line="foo = 12 + 12 .", ), TokenInfo( NUMBER, string="12", start=(1, 11), end=(1, 13), - line="foo = 12 + 12 .\n", + line="foo = 12 + 12 .", ), ] ], @@ -733,9 +733,9 @@ def test_cut(self) -> None: self.assertEqual( node, [ - TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)\n"), - TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)\n"), - TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)\n"), + TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"), + TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)"), + TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"), ], ) From 3e8c0aaff4e916174acba69ffff0c6d67f93e18e Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 24 May 2023 15:31:36 +0200 Subject: [PATCH 3/8] Fix test_pegen tests after changes in the Python tokenizer --- Lib/test/test_peg_generator/test_pegen.py | 108 +++++++++++----------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index d92da7b29bff98..2122242e9a8e8a 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -96,14 +96,14 @@ def test_gather(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2" ), TokenInfo( - NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n" + NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n" + NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2" ), ], ) @@ -119,8 +119,8 @@ def test_expr_grammar(self) -> None: self.assertEqual( node, [ - TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n"), - TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42"), + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42"), ], ) @@ -137,19 +137,19 @@ def test_optional_operator(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2" ), [ TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2" ), TokenInfo( - NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n" + NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2" ), ], ], TokenInfo( - NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n" + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2" ), ], ) @@ -158,10 +158,10 @@ def test_optional_operator(self) -> None: node, [ [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"), None, ], - TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"), ], ) @@ -178,11 +178,11 @@ def test_optional_literal(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1+" ), - TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+\n"), + TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+"), ], - TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+"), ], ) node = parse_string("1\n", parser_class) @@ -190,10 +190,10 @@ def test_optional_literal(self) -> None: node, [ [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"), None, ], - TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"), ], ) @@ -210,19 +210,19 @@ def test_alt_optional_operator(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2" ), [ TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2" ), TokenInfo( - NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2\n" + NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2" ), ], ], TokenInfo( - NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2\n" + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 + 2" ), ], ) @@ -231,10 +231,10 @@ def test_alt_optional_operator(self) -> None: node, [ [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"), None, ], - TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"), ], ) @@ -248,17 +248,17 @@ def test_repeat_0_simple(self) -> None: self.assertEqual( node, [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3"), [ TokenInfo( - NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n" + NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3" ), TokenInfo( - NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n" + NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n" + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3" ), ], ) @@ -266,9 +266,9 @@ def test_repeat_0_simple(self) -> None: self.assertEqual( node, [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1"), [], - TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"), + TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1"), ], ) @@ -283,36 +283,36 @@ def test_repeat_0_complex(self) -> None: node, [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3" ), [ [ TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="2", start=(1, 4), end=(1, 5), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], [ TokenInfo( - OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="3", start=(1, 8), end=(1, 9), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], ], TokenInfo( - NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3" ), ], ) @@ -327,17 +327,17 @@ def test_repeat_1_simple(self) -> None: self.assertEqual( node, [ - TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n"), + TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3"), [ TokenInfo( - NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n" + NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3" ), TokenInfo( - NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n" + NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n" + NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3" ), ], ) @@ -355,36 +355,36 @@ def test_repeat_1_complex(self) -> None: node, [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3" ), [ [ TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="2", start=(1, 4), end=(1, 5), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], [ TokenInfo( - OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="3", start=(1, 8), end=(1, 9), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], ], TokenInfo( - NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3" ), ], ) @@ -403,17 +403,17 @@ def test_repeat_with_sep_simple(self) -> None: [ [ TokenInfo( - NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3\n" + NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2, 3" ), TokenInfo( - NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3\n" + NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2, 3" ), TokenInfo( - NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3\n" + NUMBER, string="3", start=(1, 6), end=(1, 7), line="1, 2, 3" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3\n" + NEWLINE, string="\n", start=(1, 7), end=(1, 8), line="1, 2, 3" ), ], ) @@ -447,28 +447,28 @@ def test_left_recursive(self) -> None: string="1", start=(1, 0), end=(1, 1), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), TokenInfo( - OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n" + OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3" ), TokenInfo( NUMBER, string="2", start=(1, 4), end=(1, 5), - line="1 + 2 + 3\n", + line="1 + 2 + 3", ), ], TokenInfo( - OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n" + OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3" ), TokenInfo( - NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n" + NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3" ), ], TokenInfo( - NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n" + NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3" ), ], ) From 1c905d77cd527c69b1022f73dadfb83f294222c4 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 24 May 2023 15:34:31 +0200 Subject: [PATCH 4/8] Take fixup_build_ext from 3.11 to use in the peg-generator --- Tools/peg_generator/pegen/build.py | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 2d50f87f198d85..aace684045b9f8 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -1,4 +1,5 @@ import itertools +import os import pathlib import sys import sysconfig @@ -27,6 +28,46 @@ def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[ return f"{flags} {py_flags_nodist}".split() +def fixup_build_ext(cmd): + """Function needed to make build_ext tests pass. + + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. + + When Python was built with in debug mode on Windows, build_ext commands + need their debug attribute set, and it is not done automatically for + some reason. + + This function handles both of these things. Example use: + + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + + Unlike most other Unix platforms, Mac OS X embeds absolute paths + to shared libraries into executables, so the fixup is not needed there. + + Taken from distutils (was part of the CPython stdlib until Python 3.11) + """ + if os.name == 'nt': + cmd.debug = sys.executable.endswith('_d.exe') + elif sysconfig.get_config_var('Py_ENABLE_SHARED'): + # To further add to the shared builds fun on Unix, we can't just add + # library_dirs to the Extension() instance because that doesn't get + # plumbed through to the final compiler command. + runshared = sysconfig.get_config_var('RUNSHARED') + if runshared is None: + cmd.library_dirs = ['.'] + else: + if sys.platform == 'darwin': + cmd.library_dirs = [] + else: + name, equals, value = runshared.partition('=') + cmd.library_dirs = [d for d in value.split(os.pathsep) if d] + + + def compile_c_extension( generated_source_path: str, build_dir: Optional[str] = None, @@ -99,6 +140,7 @@ def compile_c_extension( ) dist = Distribution({"name": extension_name, "ext_modules": [extension]}) cmd = dist.get_command_obj("build_ext") + fixup_build_ext(cmd) cmd.build_lib = str(source_file_path.parent) cmd.include_dirs = include_dirs if build_dir: From 339e5922e25cb0408e2b5d20d4ba471433082e38 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 25 May 2023 16:41:05 +0200 Subject: [PATCH 5/8] Use full path in venv python executable --- Lib/test/support/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index aa3518c0da427a..f72554a5d9abd3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2291,9 +2291,9 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): if sys.executable.endswith('.exe'): python_exe += '.exe' if sys.platform == 'win32': - python = os.path.join(venv_dir, 'Scripts', python_exe) + python = os.path.join(temp_dir, venv_dir, 'Scripts', python_exe) else: - python = os.path.join(venv_dir, 'bin', python_exe) + python = os.path.join(temp_dir, venv_dir, 'bin', python_exe) cmd = [python, '-X', 'dev', '-m', 'pip', 'install', From 672e463f08c0d71631b9c74de4081b7638f9e4fe Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 25 May 2023 17:41:58 +0200 Subject: [PATCH 6/8] Add another verbose print --- Lib/test/support/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f72554a5d9abd3..8a1c4a9608596f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2286,22 +2286,27 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): print('Run:', ' '.join(cmd)) subprocess.run(cmd, check=True) + venv = os.path.join(temp_dir, venv_dir) + # Get the Python executable of the venv python_exe = 'python' if sys.executable.endswith('.exe'): python_exe += '.exe' if sys.platform == 'win32': - python = os.path.join(temp_dir, venv_dir, 'Scripts', python_exe) + python = os.path.join(venv, 'Scripts', python_exe) else: - python = os.path.join(temp_dir, venv_dir, 'bin', python_exe) + python = os.path.join(venv, 'bin', python_exe) cmd = [python, '-X', 'dev', '-m', 'pip', 'install', findfile('setuptools-67.6.1-py3-none-any.whl'), findfile('wheel-0.40.0-py3-none-any.whl')] + if verbose: + print() + print('Run:', ' '.join(cmd)) subprocess.run(cmd, check=True) - yield os.path.join(temp_dir, venv_dir), python + yield venv, python # True if Python is built with the Py_DEBUG macro defined: if From dc89e7b8c9b847498b75749e1fbbffbeba1914d6 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 26 May 2023 12:47:25 +0200 Subject: [PATCH 7/8] Use sys.executable to find name of venv executable --- Lib/test/support/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 8a1c4a9608596f..784e3547533fa7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2289,9 +2289,7 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): venv = os.path.join(temp_dir, venv_dir) # Get the Python executable of the venv - python_exe = 'python' - if sys.executable.endswith('.exe'): - python_exe += '.exe' + python_exe = os.path.basename(sys.executable) if sys.platform == 'win32': python = os.path.join(venv, 'Scripts', python_exe) else: From b3ab1a90536edb8ce93521057975b7fcb76fd241 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 26 May 2023 14:32:00 +0200 Subject: [PATCH 8/8] Check sysconfig for site-packages path --- Lib/test/support/__init__.py | 2 +- Lib/test/test_cppext.py | 12 ++++++------ Lib/test/test_peg_generator/test_c_parser.py | 12 +++++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 784e3547533fa7..7f8b1d71dbd227 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2304,7 +2304,7 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): print('Run:', ' '.join(cmd)) subprocess.run(cmd, check=True) - yield venv, python + yield python # True if Python is built with the Py_DEBUG macro defined: if diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext.py index d3dd203b43a52a..e2fedc9735079f 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext.py @@ -36,10 +36,10 @@ def test_build_cpp03(self): @support.requires_venv_with_pip() def check_build(self, std_cpp03, extension_name): venv_dir = 'env' - with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as (_, python): - self._check_build(std_cpp03, extension_name, python) + with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: + self._check_build(std_cpp03, extension_name, python_exe) - def _check_build(self, std_cpp03, extension_name, python): + def _check_build(self, std_cpp03, extension_name, python_exe): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py")) @@ -63,7 +63,7 @@ def run_cmd(operation, cmd): f"{operation} failed with exit code {proc.returncode}") # Build and install the C++ extension - cmd = [python, '-X', 'dev', + cmd = [python_exe, '-X', 'dev', '-m', 'pip', 'install', '--no-build-isolation', os.path.abspath(pkg_dir)] run_cmd('Install', cmd) @@ -71,14 +71,14 @@ def run_cmd(operation, cmd): # Do a reference run. Until we test that running python # doesn't leak references (gh-94755), run it so one can manually check # -X showrefcount results against this baseline. - cmd = [python, + cmd = [python_exe, '-X', 'dev', '-X', 'showrefcount', '-c', 'pass'] run_cmd('Reference run', cmd) # Import the C++ extension - cmd = [python, + cmd = [python_exe, '-X', 'dev', '-X', 'showrefcount', '-c', f"import {extension_name}"] diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 34c3009002fb80..af39faeba94357 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -1,4 +1,5 @@ import contextlib +import subprocess import sysconfig import textwrap import unittest @@ -90,11 +91,12 @@ def setUpClass(cls): cls.addClassCleanup(shutil.rmtree, cls.library_dir) with contextlib.ExitStack() as stack: - full_venv_path, _ = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv")) - stack.enter_context(import_helper.DirsOnSysPath(os.path.join(full_venv_path, - "lib", - f"python{sysconfig.get_python_version()}", - "site-packages"))) + python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv")) + sitepackages = subprocess.check_output( + [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], + text=True, + ).strip() + stack.enter_context(import_helper.DirsOnSysPath(sitepackages)) cls.addClassCleanup(stack.pop_all().close) @support.requires_venv_with_pip()